From 4f14fafa34530d740f4695f4629fa594b322c448 Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Wed, 24 Jul 2024 11:59:35 +0530 Subject: [PATCH 001/211] fix(panorama): update panorama checking mechanism to reduce checks --- .../models/file/extensions/file_props.dart | 12 +- mobile/lib/models/file/file.dart | 11 ++ mobile/lib/models/metadata/file_magic.dart | 8 + mobile/lib/ui/actions/file/file_actions.dart | 5 + .../ui/tools/editor/video_editor_page.dart | 4 +- .../lib/ui/viewer/file/file_bottom_bar.dart | 177 +++++++++-------- .../viewer/file/panorama_viewer_screen.dart | 185 ++++++++++++++++-- mobile/lib/utils/exif_util.dart | 6 + mobile/lib/utils/file_uploader.dart | 2 + mobile/lib/utils/file_uploader_util.dart | 3 + mobile/lib/utils/panorama_util.dart | 52 ++++- 11 files changed, 348 insertions(+), 117 deletions(-) diff --git a/mobile/lib/models/file/extensions/file_props.dart b/mobile/lib/models/file/extensions/file_props.dart index 6f68f916b9..9be9bd4c11 100644 --- a/mobile/lib/models/file/extensions/file_props.dart +++ b/mobile/lib/models/file/extensions/file_props.dart @@ -18,18 +18,26 @@ extension FilePropsExtn on EnteFile { bool get hasDims => height > 0 && width > 0; - // return true if the file is a panorama image, null if the dimensions are not available + // return true if the file can be a panorama image, null if the dimensions are not available bool? isPanorama() { if (fileType != FileType.image) { return false; } + if (pubMagicMetadata?.mediaType != null) { + return (pubMagicMetadata!.mediaType! & 1) == 1; + } + return null; + } + + bool canBePanorama() { if (hasDims) { + if (height < 8000 && width < 8000) return false; if (height > width) { return height / width >= 2.0; } return width / height >= 2.0; } - return null; + return false; } bool get canEditMetaInfo => isUploaded && isOwner; diff --git a/mobile/lib/models/file/file.dart b/mobile/lib/models/file/file.dart index 9df25bb051..b4276878ce 100644 --- a/mobile/lib/models/file/file.dart +++ b/mobile/lib/models/file/file.dart @@ -2,6 +2,7 @@ import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:logging/logging.dart'; +import "package:motion_photos/src/xmp_extractor.dart"; import 'package:path/path.dart'; import 'package:photo_manager/photo_manager.dart'; import 'package:photos/core/configuration.dart'; @@ -13,6 +14,7 @@ import "package:photos/service_locator.dart"; import 'package:photos/utils/date_time_util.dart'; import 'package:photos/utils/exif_util.dart'; import 'package:photos/utils/file_uploader_util.dart'; +import "package:photos/utils/panorama_util.dart"; //Todo: files with no location data have lat and long set to 0.0. This should ideally be null. class EnteFile { @@ -179,6 +181,15 @@ class EnteFile { hasExifTime = true; creationTime = exifTime.microsecondsSinceEpoch; } + mediaUploadData.isPanorama = checkPanoramaFromEXIF(null, exifData); + + try { + final xmpData = XMPExtractor() + .extract(mediaUploadData.sourceFile!.readAsBytesSync()); + mediaUploadData.isPanorama = checkPanoramaFromXMP(xmpData); + } catch (_) {} + + mediaUploadData.isPanorama ??= false; } if (Platform.isAndroid) { //Fix for missing location data in lower android versions. diff --git a/mobile/lib/models/metadata/file_magic.dart b/mobile/lib/models/metadata/file_magic.dart index d50dc0dd84..6c146fa031 100644 --- a/mobile/lib/models/metadata/file_magic.dart +++ b/mobile/lib/models/metadata/file_magic.dart @@ -8,6 +8,7 @@ const captionKey = "caption"; const uploaderNameKey = "uploaderName"; const widthKey = 'w'; const heightKey = 'h'; +const mediaTypeKey = 'mediaType'; const latKey = "lat"; const longKey = "long"; const motionVideoIndexKey = "mvi"; @@ -55,6 +56,11 @@ class PubMagicMetadata { // should have exact same hash with should match the constant `blackThumbnailBase64` bool? noThumb; + // null -> not computed + // 0 -> normal + // 1 -> panorama + int? mediaType; + PubMagicMetadata({ this.editedTime, this.editedName, @@ -66,6 +72,7 @@ class PubMagicMetadata { this.long, this.mvi, this.noThumb, + this.mediaType, }); factory PubMagicMetadata.fromEncodedJson(String encodedJson) => @@ -87,6 +94,7 @@ class PubMagicMetadata { long: map[longKey], mvi: map[motionVideoIndexKey], noThumb: map[noThumbKey], + mediaType: map[mediaTypeKey], ); } } diff --git a/mobile/lib/ui/actions/file/file_actions.dart b/mobile/lib/ui/actions/file/file_actions.dart index 83ba1fa72a..6a7305a019 100644 --- a/mobile/lib/ui/actions/file/file_actions.dart +++ b/mobile/lib/ui/actions/file/file_actions.dart @@ -5,6 +5,7 @@ import "package:modal_bottom_sheet/modal_bottom_sheet.dart"; import "package:photos/generated/l10n.dart"; import 'package:photos/models/file/file.dart'; import 'package:photos/models/file/file_type.dart'; +import "package:photos/service_locator.dart"; import "package:photos/theme/colors.dart"; import "package:photos/theme/ente_theme.dart"; import "package:photos/ui/components/action_sheet_widget.dart"; @@ -13,6 +14,7 @@ import "package:photos/ui/components/models/button_type.dart"; import 'package:photos/ui/viewer/file/file_details_widget.dart'; import "package:photos/utils/delete_file_util.dart"; import "package:photos/utils/dialog_util.dart"; +import "package:photos/utils/panorama_util.dart"; import "package:photos/utils/toast_util.dart"; Future showSingleFileDeleteSheet( @@ -135,6 +137,9 @@ Future showSingleFileDeleteSheet( } Future showDetailsSheet(BuildContext context, EnteFile file) async { + if (flagService.internalUser) { + guardedCheckPanorama(file).ignore(); + } final colorScheme = getEnteColorScheme(context); return showBarModalBottomSheet( topControl: const SizedBox.shrink(), diff --git a/mobile/lib/ui/tools/editor/video_editor_page.dart b/mobile/lib/ui/tools/editor/video_editor_page.dart index 5cb5afac8a..2fcba8990b 100644 --- a/mobile/lib/ui/tools/editor/video_editor_page.dart +++ b/mobile/lib/ui/tools/editor/video_editor_page.dart @@ -200,7 +200,9 @@ class _VideoEditorPageState extends State { final config = VideoFFmpegVideoEditorConfig( _controller!, - format: VideoExportFormat.mp4, + format: VideoExportFormat( + path.extension(widget.ioFile.path).substring(1), + ), // commandBuilder: (config, videoPath, outputPath) { // final List filters = config.getExportFilters(); // filters.add('hflip'); // add horizontal flip diff --git a/mobile/lib/ui/viewer/file/file_bottom_bar.dart b/mobile/lib/ui/viewer/file/file_bottom_bar.dart index 9db7275918..9d9b00ebb6 100644 --- a/mobile/lib/ui/viewer/file/file_bottom_bar.dart +++ b/mobile/lib/ui/viewer/file/file_bottom_bar.dart @@ -50,7 +50,6 @@ class FileBottomBarState extends State { bool _isFileSwipeLocked = false; late final StreamSubscription _fileSwipeLockEventSubscription; - bool isPanorama = false; int? lastFileGenID; @override @@ -73,26 +72,16 @@ class FileBottomBarState extends State { @override Widget build(BuildContext context) { if (flagService.internalUser) { - isPanorama = widget.file.isPanorama() ?? false; - _checkPanorama(); + if (widget.file.canBePanorama()) { + lastFileGenID = widget.file.generatedID; + if (lastFileGenID != widget.file.generatedID) { + guardedCheckPanorama(widget.file).ignore(); + } + } } return _getBottomBar(); } - // _checkPanorama() method is used to check if the file is a panorama image. - // This handles the case when the the file dims (width and height) are not available. - Future _checkPanorama() async { - if (lastFileGenID == widget.file.generatedID) { - return; - } - lastFileGenID = widget.file.generatedID; - final result = await checkIfPanorama(widget.file); - if (mounted && isPanorama == !result) { - isPanorama = result; - setState(() {}); - } - } - void safeRefresh() { if (mounted) { setState(() {}); @@ -175,25 +164,6 @@ class FileBottomBarState extends State { ); } - if (isPanorama) { - children.add( - Tooltip( - message: S.of(context).panorama, - child: Padding( - padding: const EdgeInsets.only(top: 12, bottom: 12), - child: IconButton( - icon: const Icon( - Icons.threesixty, - color: Colors.white, - ), - onPressed: () async { - await openPanoramaViewerPage(widget.file); - }, - ), - ), - ), - ); - } children.add( Tooltip( message: S.of(context).share, @@ -227,59 +197,92 @@ class FileBottomBarState extends State { curve: Curves.easeInOut, child: Align( alignment: Alignment.bottomCenter, - child: Container( - decoration: BoxDecoration( - gradient: LinearGradient( - begin: Alignment.topCenter, - end: Alignment.bottomCenter, - colors: [ - Colors.transparent, - Colors.black.withOpacity(0.6), - Colors.black.withOpacity(0.72), - ], - stops: const [0, 0.8, 1], - ), - ), - child: Padding( - padding: EdgeInsets.only(bottom: safeAreaBottomPadding), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - widget.file.caption?.isNotEmpty ?? false - ? Padding( - padding: const EdgeInsets.fromLTRB( - 16, - 12, - 16, - 0, - ), - child: GestureDetector( - onTap: () async { - await _displayDetails(widget.file); - await Future.delayed( - const Duration(milliseconds: 500), - ); //Waiting for some time till the caption gets updated in db if the user closes the bottom sheet without pressing 'done' - safeRefresh(); - }, - child: Text( - widget.file.caption!, - overflow: TextOverflow.ellipsis, - maxLines: 1, - style: getEnteTextTheme(context) - .mini - .copyWith(color: textBaseDark), - textAlign: TextAlign.center, - ), - ), - ) - : const SizedBox.shrink(), - Row( - mainAxisAlignment: MainAxisAlignment.spaceAround, - children: children, + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + if (widget.file.isPanorama() == true) + Align( + alignment: Alignment.centerRight, + child: Tooltip( + message: S.of(context).panorama, + child: Padding( + padding: const EdgeInsets.only( + top: 12, + bottom: 12, + right: 20, + ), + child: IconButton( + style: IconButton.styleFrom( + backgroundColor: const Color(0xFF252525), + fixedSize: const Size(44, 44), + ), + icon: const Icon( + Icons.vrpano_outlined, + color: Colors.white, + size: 26, + ), + onPressed: () async { + await openPanoramaViewerPage(widget.file); + }, + ), + ), ), - ], + ), + Container( + decoration: BoxDecoration( + gradient: LinearGradient( + begin: Alignment.topCenter, + end: Alignment.bottomCenter, + colors: [ + Colors.transparent, + Colors.black.withOpacity(0.6), + Colors.black.withOpacity(0.72), + ], + stops: const [0, 0.8, 1], + ), + ), + child: Padding( + padding: EdgeInsets.only(bottom: safeAreaBottomPadding), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + widget.file.caption?.isNotEmpty ?? false + ? Padding( + padding: const EdgeInsets.fromLTRB( + 16, + 12, + 16, + 0, + ), + child: GestureDetector( + onTap: () async { + await _displayDetails(widget.file); + await Future.delayed( + const Duration(milliseconds: 500), + ); //Waiting for some time till the caption gets updated in db if the user closes the bottom sheet without pressing 'done' + safeRefresh(); + }, + child: Text( + widget.file.caption!, + overflow: TextOverflow.ellipsis, + maxLines: 1, + style: getEnteTextTheme(context) + .mini + .copyWith(color: textBaseDark), + textAlign: TextAlign.center, + ), + ), + ) + : const SizedBox.shrink(), + Row( + mainAxisAlignment: MainAxisAlignment.spaceAround, + children: children, + ), + ], + ), + ), ), - ), + ], ), ), ), diff --git a/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart b/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart index fe5c94652f..4b05bbd6f9 100644 --- a/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart +++ b/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart @@ -1,9 +1,13 @@ +import "dart:async"; import "dart:io"; import "package:flutter/material.dart"; +import "package:flutter/services.dart"; +import "package:motion_photos/src/xmp_extractor.dart"; import "package:panorama_viewer/panorama_viewer.dart"; +import "package:photos/generated/l10n.dart"; -class PanoramaViewerScreen extends StatelessWidget { +class PanoramaViewerScreen extends StatefulWidget { const PanoramaViewerScreen({ super.key, required this.file, @@ -11,26 +15,173 @@ class PanoramaViewerScreen extends StatelessWidget { final File file; + @override + State createState() => _PanoramaViewerScreenState(); +} + +class _PanoramaViewerScreenState extends State { + double width = 1.0; + double height = 1.0; + Rect croppedRect = const Rect.fromLTWH(0.0, 0.0, 1.0, 1.0); + SensorControl control = SensorControl.none; + Timer? timer; + bool isVisible = true; + + @override + void initState() { + initTimer(); + init(); + super.initState(); + } + + void initTimer() { + timer = Timer(const Duration(seconds: 5), () { + setState(() { + isVisible = false; + }); + SystemChrome.setEnabledSystemUIMode(SystemUiMode.immersiveSticky); + }); + SystemChrome.setEnabledSystemUIMode(SystemUiMode.edgeToEdge); + } + + Future init() async { + final data = XMPExtractor().extract(widget.file.readAsBytesSync()); + double? cWidth = + double.tryParse(data["GPano:CroppedAreaImageWidthPixels"] ?? ""); + double? cHeight = + double.tryParse(data["GPano:CroppedAreaImageHeightPixels"] ?? ""); + double? fWidth = double.tryParse(data["GPano:FullPanoWidthPixels"] ?? ""); + double? fHeight = double.tryParse(data["GPano:FullPanoHeightPixels"] ?? ""); + double? cLeft = double.tryParse(data["GPano:CroppedAreaLeftPixels"] ?? ""); + double? cTop = double.tryParse(data["GPano:CroppedAreaTopPixels"] ?? ""); + + // handle missing `fullPanoHeight` (e.g. Samsung camera app panorama mode) + if (fHeight == null && fWidth != null && cHeight != null) { + fHeight = (fWidth / 2).round().toDouble(); + cTop = ((fHeight - cHeight) / 2).round().toDouble(); + } + + // handle inconsistent sizing (e.g. rotated image taken with OnePlus EB2103) + if (cHeight != null && fWidth != null && fHeight != null) { + final croppedOrientation = + cWidth! > cHeight ? Orientation.landscape : Orientation.portrait; + final fullOrientation = + fWidth > fHeight ? Orientation.landscape : Orientation.portrait; + var inconsistent = false; + if (croppedOrientation != fullOrientation) { + // inconsistent orientation + inconsistent = true; + final tmp = cHeight; + cHeight = cWidth; + cWidth = tmp; + } + + if (cWidth > fWidth) { + // inconsistent full/cropped width + inconsistent = true; + final tmp = fWidth; + fWidth = cWidth; + cWidth = tmp; + } + + if (cHeight > fHeight) { + // inconsistent full/cropped height + inconsistent = true; + final tmp = cHeight; + cHeight = fHeight; + fHeight = tmp; + } + + if (inconsistent) { + cLeft = ((fWidth - cWidth) ~/ 2).toDouble(); + cTop = ((fHeight - cHeight) ~/ 2).toDouble(); + } + } + + Rect? croppedAreaRect; + if (cLeft != null && cTop != null && cWidth != null && cHeight != null) { + croppedAreaRect = Rect.fromLTWH( + cLeft.toDouble(), + cTop.toDouble(), + cWidth.toDouble(), + cHeight.toDouble(), + ); + } + + if (croppedAreaRect == null || fWidth == null || fHeight == null) return; + width = fWidth.toDouble(); + height = fHeight.toDouble(); + croppedRect = croppedAreaRect; + + setState(() {}); + } + @override Widget build(BuildContext context) { return Scaffold( - appBar: AppBar( - elevation: 0, // Remove shadow - leading: IconButton( - icon: const Icon( - Icons.arrow_back, - size: 18, + body: Stack( + children: [ + PanoramaViewer( + onTap: (_, __, ___) { + setState(() { + if (isVisible) { + timer?.cancel(); + SystemChrome.setEnabledSystemUIMode( + SystemUiMode.immersiveSticky, + ); + } else { + initTimer(); + } + isVisible = !isVisible; + }); + }, + croppedArea: croppedRect, + croppedFullWidth: width, + croppedFullHeight: height, + sensorControl: control, + child: Image.file( + widget.file, + ), ), - onPressed: () { - Navigator.of(context).pop(); - }, - ), - ), - body: PanoramaViewer( - child: Image.file( - file, - fit: BoxFit.cover, - ), + Visibility( + visible: isVisible, + child: Align( + alignment: Alignment.bottomRight, + child: Tooltip( + message: S.of(context).panorama, + child: Padding( + padding: const EdgeInsets.only( + top: 12, + bottom: 32, + right: 20, + ), + child: IconButton( + style: IconButton.styleFrom( + backgroundColor: const Color(0xFF252525), + fixedSize: const Size(44, 44), + ), + icon: Icon( + Icons.explore, + color: control != SensorControl.none + ? Colors.white + : Colors.white.withOpacity(0.4), + size: 26, + ), + onPressed: () async { + if (control != SensorControl.none) { + control = SensorControl.none; + } else { + control = SensorControl.orientation; + } + + setState(() {}); + }, + ), + ), + ), + ), + ), + ], ), ); } diff --git a/mobile/lib/utils/exif_util.dart b/mobile/lib/utils/exif_util.dart index d9bac85e6d..8243b3a92f 100644 --- a/mobile/lib/utils/exif_util.dart +++ b/mobile/lib/utils/exif_util.dart @@ -102,6 +102,12 @@ Future getVideoPropsAsync(File originalFile) async { } } +bool? checkPanoramaFromEXIF(File? file, Map? exifData) { + final element = exifData?["EXIF CustomRendered"]; + if (element?.printable == null) return null; + return element?.printable == "6"; +} + Future getCreationTimeFromEXIF( File? file, Map? exifData, diff --git a/mobile/lib/utils/file_uploader.dart b/mobile/lib/utils/file_uploader.dart index a5750f6b7a..6e81e5acc5 100644 --- a/mobile/lib/utils/file_uploader.dart +++ b/mobile/lib/utils/file_uploader.dart @@ -675,6 +675,8 @@ class FileUploader { (mediaUploadData.width ?? 0) != 0) { pubMetadata[heightKey] = mediaUploadData.height; pubMetadata[widthKey] = mediaUploadData.width; + pubMetadata[mediaTypeKey] = + mediaUploadData.isPanorama == true ? 1 : 0; } if (mediaUploadData.motionPhotoStartIndex != null) { pubMetadata[motionVideoIndexKey] = diff --git a/mobile/lib/utils/file_uploader_util.dart b/mobile/lib/utils/file_uploader_util.dart index ef4f042b7f..e443b1ad8c 100644 --- a/mobile/lib/utils/file_uploader_util.dart +++ b/mobile/lib/utils/file_uploader_util.dart @@ -44,6 +44,8 @@ class MediaUploadData { // For iOS, this value will be always null. final int? motionPhotoStartIndex; + bool? isPanorama; + MediaUploadData( this.sourceFile, this.thumbnail, @@ -52,6 +54,7 @@ class MediaUploadData { this.height, this.width, this.motionPhotoStartIndex, + this.isPanorama, }); } diff --git a/mobile/lib/utils/panorama_util.dart b/mobile/lib/utils/panorama_util.dart index d4fee93669..c002873185 100644 --- a/mobile/lib/utils/panorama_util.dart +++ b/mobile/lib/utils/panorama_util.dart @@ -1,7 +1,10 @@ -import "package:flutter/painting.dart"; +import "package:motion_photos/src/xmp_extractor.dart"; import "package:photos/models/file/extensions/file_props.dart"; import "package:photos/models/file/file.dart"; import "package:photos/models/file/file_type.dart"; +import "package:photos/models/metadata/file_magic.dart"; +import "package:photos/services/file_magic_service.dart"; +import "package:photos/utils/exif_util.dart"; import "package:photos/utils/file_util.dart"; /// Check if the file is a panorama image. @@ -9,20 +12,49 @@ Future checkIfPanorama(EnteFile enteFile) async { if (enteFile.fileType != FileType.image) { return false; } - if (enteFile.isPanorama() != null) { - return enteFile.isPanorama()!; - } final file = await getFile(enteFile); if (file == null) { return false; } + try { + final result = XMPExtractor().extract(file.readAsBytesSync()); + if (result["GPano:ProjectionType"] == "cylindrical" || + result["GPano:ProjectionType"] == "equirectangular") { + return true; + } + } catch (_) {} + + final result = await readExifAsync(file); + + final element = result["EXIF CustomRendered"]; + return element?.printable == "6"; +} + +bool? checkPanoramaFromXMP(Map xmpData) { + if (xmpData["GPano:ProjectionType"] == "cylindrical" || + xmpData["GPano:ProjectionType"] == "equirectangular") { + return true; + } + return false; +} + +// guardedCheckPanorama() method is used to check if the file is a panorama image. +Future guardedCheckPanorama(EnteFile file) async { + if (file.isPanorama() != null) { + return; + } + final result = await checkIfPanorama(file); + + // Update the metadata if it is not updated + if (file.isPanorama() == null && file.canEditMetaInfo) { + int? mediaType = file.pubMagicMetadata?.mediaType; + mediaType ??= 0; - final image = await decodeImageFromList(await file.readAsBytes()); - final width = image.width.toDouble(); - final height = image.height.toDouble(); + mediaType = mediaType | (result ? 1 : 0); - if (height > width) { - return height / width >= 2.0; + FileMagicService.instance.updatePublicMagicMetadata( + [file], + {mediaTypeKey: mediaType}, + ).ignore(); } - return width / height >= 2.0; } From 198dab9f58092166353408d151d9029f2b4e6f2e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 29 Jul 2024 10:37:13 +0530 Subject: [PATCH 002/211] [server] Add db script to store data --- .../migrations/89_derived_data_table.down.sql | 7 +++++ .../migrations/89_derived_data_table.up.sql | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 server/migrations/89_derived_data_table.down.sql create mode 100644 server/migrations/89_derived_data_table.up.sql diff --git a/server/migrations/89_derived_data_table.down.sql b/server/migrations/89_derived_data_table.down.sql new file mode 100644 index 0000000000..c71f42880f --- /dev/null +++ b/server/migrations/89_derived_data_table.down.sql @@ -0,0 +1,7 @@ + +DROP INDEX IF EXISTS idx_derived_user_id_updated_at; +DROP INDEX IF EXISTS idx_derived_user_id_data_type; + +DROP TABLE IF EXISTS derived; + +DROP TYPE IF EXISTS derived_data_type; \ No newline at end of file diff --git a/server/migrations/89_derived_data_table.up.sql b/server/migrations/89_derived_data_table.up.sql new file mode 100644 index 0000000000..fbbb15146a --- /dev/null +++ b/server/migrations/89_derived_data_table.up.sql @@ -0,0 +1,27 @@ +-- Create the data_type enum +CREATE TYPE derived_data_type AS ENUM ('img_jpg_preview', 'vid_hls_preview', 'meta'); + +-- Create the derived table +CREATE TABLE derived ( + file_id BIGINT NOT NULL, + user_id BIGINT NOT NULL, + data_type derived_data_type NOT NULL, + size BIGINT NOT NULL, + latest_bucket s3region NOT NULL, + replicated_buckets s3region[] NOT NULL, +-- following field contains list of buckets from where we need to delete the data as the given data_type will not longer be persisted in that dc + delete_from_buckets s3region[] NOT NULL DEFAULT '{}', + pending_sync BOOLEAN NOT NULL DEFAULT false, + created_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), + updated_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), + PRIMARY KEY (file_id, data_type) +); + +-- Add primary key +ALTER TABLE derived ADD PRIMARY KEY (file_id, data_type); + +-- Add index for user_id and data_type +CREATE INDEX idx_derived_user_id_data_type ON derived (user_id, data_type); + +-- Add index for user_id and updated_at for efficient querying +CREATE INDEX idx_derived_user_id_updated_at ON derived (user_id, updated_at); \ No newline at end of file From 950b2bb997d13af1af21fe6dc8079c160b145fe9 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 29 Jul 2024 11:42:14 +0530 Subject: [PATCH 003/211] [server] Update db script --- server/migrations/89_derived_data_table.down.sql | 8 ++++---- server/migrations/89_derived_data_table.up.sql | 13 +++++++------ 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/server/migrations/89_derived_data_table.down.sql b/server/migrations/89_derived_data_table.down.sql index c71f42880f..a6f4e7e3d0 100644 --- a/server/migrations/89_derived_data_table.down.sql +++ b/server/migrations/89_derived_data_table.down.sql @@ -1,7 +1,7 @@ -DROP INDEX IF EXISTS idx_derived_user_id_updated_at; -DROP INDEX IF EXISTS idx_derived_user_id_data_type; +DROP INDEX IF EXISTS idx_file_data_user_id_data_type; +DROP INDEX IF EXISTS idx_file_data_user_id_updated_at; -DROP TABLE IF EXISTS derived; +DROP TABLE IF EXISTS file_data; -DROP TYPE IF EXISTS derived_data_type; \ No newline at end of file +DROP TYPE IF EXISTS file_data_type; \ No newline at end of file diff --git a/server/migrations/89_derived_data_table.up.sql b/server/migrations/89_derived_data_table.up.sql index fbbb15146a..4ba869c71f 100644 --- a/server/migrations/89_derived_data_table.up.sql +++ b/server/migrations/89_derived_data_table.up.sql @@ -1,27 +1,28 @@ -- Create the data_type enum -CREATE TYPE derived_data_type AS ENUM ('img_jpg_preview', 'vid_hls_preview', 'meta'); +CREATE TYPE file_data_type AS ENUM ('img_jpg_preview', 'vid_hls_preview', 'derived'); -- Create the derived table -CREATE TABLE derived ( +CREATE TABLE file_data ( file_id BIGINT NOT NULL, user_id BIGINT NOT NULL, - data_type derived_data_type NOT NULL, + data_type file_data_type NOT NULL, size BIGINT NOT NULL, latest_bucket s3region NOT NULL, replicated_buckets s3region[] NOT NULL, -- following field contains list of buckets from where we need to delete the data as the given data_type will not longer be persisted in that dc delete_from_buckets s3region[] NOT NULL DEFAULT '{}', pending_sync BOOLEAN NOT NULL DEFAULT false, + last_sync_time BIGINT NOT NULL DEFAULT 0, created_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), updated_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), PRIMARY KEY (file_id, data_type) ); -- Add primary key -ALTER TABLE derived ADD PRIMARY KEY (file_id, data_type); +ALTER TABLE file_data ADD PRIMARY KEY (file_id, data_type); -- Add index for user_id and data_type -CREATE INDEX idx_derived_user_id_data_type ON derived (user_id, data_type); +CREATE INDEX idx_file_data_user_id_data_type ON file_data (user_id, data_type); -- Add index for user_id and updated_at for efficient querying -CREATE INDEX idx_derived_user_id_updated_at ON derived (user_id, updated_at); \ No newline at end of file +CREATE INDEX idx_file_data_user_id_updated_at ON file_data (user_id, updated_at); \ No newline at end of file From 2cc87140edfa5877021b85a39d8954532fe8cdbd Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:37:58 +0530 Subject: [PATCH 004/211] Add basic endpoint to unblock testing --- server/cmd/museum/main.go | 6 + server/ente/fileobjects/type.go | 45 +++++ .../migrations/89_derived_data_table.up.sql | 3 +- server/pkg/api/file.go | 46 +++++ server/pkg/api/file_preview.go | 1 + server/pkg/controller/file.go | 2 + server/pkg/controller/file_preview.go | 156 ++++++++++++++++ server/pkg/controller/filedata/controller.go | 4 + server/pkg/controller/filedata/file_object.go | 5 + server/pkg/controller/preview/controller.go | 8 + server/pkg/repo/filedata/repository.go | 172 ++++++++++++++++++ 11 files changed, 446 insertions(+), 2 deletions(-) create mode 100644 server/ente/fileobjects/type.go create mode 100644 server/pkg/api/file_preview.go create mode 100644 server/pkg/controller/file_preview.go create mode 100644 server/pkg/controller/filedata/controller.go create mode 100644 server/pkg/controller/filedata/file_object.go create mode 100644 server/pkg/controller/preview/controller.go create mode 100644 server/pkg/repo/filedata/repository.go diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index 9258fa9b77..3fb5359289 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -408,6 +408,12 @@ func main() { privateAPI.GET("/files/download/v2/:fileID", fileHandler.Get) privateAPI.GET("/files/preview/:fileID", fileHandler.GetThumbnail) privateAPI.GET("/files/preview/v2/:fileID", fileHandler.GetThumbnail) + + privateAPI.GET("/files/file-data/playlist/:fileID", fileHandler.GetVideoPlaylist) + privateAPI.POST("/files/file-data/playlist", fileHandler.ReportVideoPlayList) + privateAPI.GET("/files/file-data/preview/upload-url/:fileID", fileHandler.GetVideoUploadURL) + privateAPI.GET("/files/file-data/preview/:fileID", fileHandler.GetVideoUploadURL) + privateAPI.POST("/files", fileHandler.CreateOrUpdate) privateAPI.POST("/files/copy", fileHandler.CopyFiles) privateAPI.PUT("/files/update", fileHandler.Update) diff --git a/server/ente/fileobjects/type.go b/server/ente/fileobjects/type.go new file mode 100644 index 0000000000..8c5ce65919 --- /dev/null +++ b/server/ente/fileobjects/type.go @@ -0,0 +1,45 @@ +package fileobjects + +import ( + "database/sql/driver" + "errors" + "fmt" +) + +type Type string + +const ( + OriginalFile Type = "file" + OriginalThumbnail Type = "thumb" + PreviewImage Type = "previewImage" + PreviewVideo Type = "previewVideo" + Derived Type = "derived" +) + +func (ft Type) IsValid() bool { + switch ft { + case OriginalFile, OriginalThumbnail, PreviewImage, PreviewVideo, Derived: + return true + } + return false +} + +func (ft *Type) Scan(value interface{}) error { + strValue, ok := value.(string) + if !ok { + return errors.New("type should be a string") + } + + *ft = Type(strValue) + if !ft.IsValid() { + return fmt.Errorf("invalid FileType value: %s", strValue) + } + return nil +} + +func (ft Type) Value() (driver.Value, error) { + if !ft.IsValid() { + return nil, fmt.Errorf("invalid FileType value: %s", ft) + } + return string(ft), nil +} diff --git a/server/migrations/89_derived_data_table.up.sql b/server/migrations/89_derived_data_table.up.sql index 4ba869c71f..548af5f12a 100644 --- a/server/migrations/89_derived_data_table.up.sql +++ b/server/migrations/89_derived_data_table.up.sql @@ -12,14 +12,13 @@ CREATE TABLE file_data ( -- following field contains list of buckets from where we need to delete the data as the given data_type will not longer be persisted in that dc delete_from_buckets s3region[] NOT NULL DEFAULT '{}', pending_sync BOOLEAN NOT NULL DEFAULT false, + is_deleted BOOLEAN NOT NULL DEFAULT false, last_sync_time BIGINT NOT NULL DEFAULT 0, created_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), updated_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), PRIMARY KEY (file_id, data_type) ); --- Add primary key -ALTER TABLE file_data ADD PRIMARY KEY (file_id, data_type); -- Add index for user_id and data_type CREATE INDEX idx_file_data_user_id_data_type ON file_data (user_id, data_type); diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go index 064bc3be08..94760049af 100644 --- a/server/pkg/api/file.go +++ b/server/pkg/api/file.go @@ -120,6 +120,52 @@ func (h *FileHandler) GetUploadURLs(c *gin.Context) { }) } +func (h *FileHandler) GetVideoUploadURL(c *gin.Context) { + enteApp := auth.GetApp(c) + userID, fileID := getUserAndFileIDs(c) + urls, err := h.Controller.GetVideoUploadUrl(c, userID, fileID, enteApp) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.JSON(http.StatusOK, urls) +} + +func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) { + userID, fileID := getUserAndFileIDs(c) + url, err := h.Controller.GetPreviewUrl(c, userID, fileID) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.Redirect(http.StatusTemporaryRedirect, url) +} + +func (h *FileHandler) ReportVideoPlayList(c *gin.Context) { + var request ente.InsertOrUpdateEmbeddingRequest + if err := c.ShouldBindJSON(&request); err != nil { + handler.Error(c, + stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) + return + } + err := h.Controller.ReportVideoPreview(c, request) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.Status(http.StatusOK) +} + +func (h *FileHandler) GetVideoPlaylist(c *gin.Context) { + fileID, _ := strconv.ParseInt(c.Param("fileID"), 10, 64) + response, err := h.Controller.GetPlaylist(c, fileID) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.JSON(http.StatusOK, response) +} + // GetMultipartUploadURLs returns an array of PartUpload PresignedURLs func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) { enteApp := auth.GetApp(c) diff --git a/server/pkg/api/file_preview.go b/server/pkg/api/file_preview.go new file mode 100644 index 0000000000..778f64ec17 --- /dev/null +++ b/server/pkg/api/file_preview.go @@ -0,0 +1 @@ +package api diff --git a/server/pkg/controller/file.go b/server/pkg/controller/file.go index 5bb4f47bf9..3d51b6f2db 100644 --- a/server/pkg/controller/file.go +++ b/server/pkg/controller/file.go @@ -284,6 +284,8 @@ func (c *FileController) GetUploadURLs(ctx context.Context, userID int64, count return urls, nil } + + // GetFileURL verifies permissions and returns a presigned url to the requested file func (c *FileController) GetFileURL(ctx *gin.Context, userID int64, fileID int64) (string, error) { err := c.verifyFileAccess(userID, fileID) diff --git a/server/pkg/controller/file_preview.go b/server/pkg/controller/file_preview.go new file mode 100644 index 0000000000..1aa3bfb1dc --- /dev/null +++ b/server/pkg/controller/file_preview.go @@ -0,0 +1,156 @@ +package controller + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/ente-io/museum/ente" + "github.com/ente-io/museum/pkg/utils/auth" + "github.com/ente-io/museum/pkg/utils/network" + "github.com/ente-io/stacktrace" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + "strconv" + "strings" +) + +const ( + _model = "hls_video" +) + +// GetUploadURLs returns a bunch of presigned URLs for uploading files +func (c *FileController) GetVideoUploadUrl(ctx context.Context, userID int64, fileID int64, app ente.App) (*ente.UploadURL, error) { + err := c.UsageCtrl.CanUploadFile(ctx, userID, nil, app) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + s3Client := c.S3Config.GetDerivedStorageS3Client() + dc := c.S3Config.GetDerivedStorageDataCenter() + bucket := c.S3Config.GetDerivedStorageBucket() + objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/" + _model + url, err := c.getObjectURL(s3Client, dc, bucket, objectKey) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + log.Infof("Got upload URL for %s", objectKey) + return &url, nil +} + +func (c *FileController) GetPreviewUrl(ctx context.Context, userID int64, fileID int64) (string, error) { + err := c.verifyFileAccess(userID, fileID) + if err != nil { + return "", err + } + objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video" + s3Client := c.S3Config.GetDerivedStorageS3Client() + r, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: c.S3Config.GetDerivedStorageBucket(), + Key: &objectKey, + }) + return r.Presign(PreSignedRequestValidityDuration) +} + +func (c *FileController) GetPlaylist(ctx *gin.Context, fileID int64) (ente.EmbeddingObject, error) { + objectKey := strconv.FormatInt(auth.GetUserID(ctx.Request.Header), 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video_playlist.m3u8" + // check if object exists + err := c.checkObjectExists(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter()) + if err != nil { + return ente.EmbeddingObject{}, stacktrace.Propagate(ente.NewBadRequestWithMessage("Video playlist does not exist"), fmt.Sprintf("objectKey: %s", objectKey)) + } + return c.downloadObject(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter()) +} + +func (c *FileController) ReportVideoPreview(ctx *gin.Context, req ente.InsertOrUpdateEmbeddingRequest) error { + userID := auth.GetUserID(ctx.Request.Header) + if strings.Compare(req.Model, "hls_video") != 0 { + return stacktrace.Propagate(ente.NewBadRequestWithMessage("Model should be hls_video"), "Invalid fileID") + } + count, err := c.CollectionRepo.GetCollectionCount(req.FileID) + if err != nil { + return stacktrace.Propagate(err, "") + } + if count < 1 { + return stacktrace.Propagate(ente.ErrNotFound, "") + } + version := 1 + if req.Version != nil { + version = *req.Version + } + objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(req.FileID, 10) + "/hls_video" + playlistKey := objectKey + "_playlist.m3u8" + + // verify that objectKey exists + err = c.checkObjectExists(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter()) + if err != nil { + return stacktrace.Propagate(ente.NewBadRequestWithMessage("Video object does not exist, upload that before playlist reporting"), fmt.Sprintf("objectKey: %s", objectKey)) + } + + obj := ente.EmbeddingObject{ + Version: version, + EncryptedEmbedding: req.EncryptedEmbedding, + DecryptionHeader: req.DecryptionHeader, + Client: network.GetClientInfo(ctx), + } + _, uploadErr := c.uploadObject(obj, playlistKey, c.S3Config.GetDerivedStorageDataCenter()) + if uploadErr != nil { + log.Error(uploadErr) + return stacktrace.Propagate(uploadErr, "") + } + return nil +} + +func (c *FileController) uploadObject(obj ente.EmbeddingObject, key string, dc string) (int, error) { + embeddingObj, _ := json.Marshal(obj) + s3Client := c.S3Config.GetS3Client(dc) + s3Bucket := c.S3Config.GetBucket(dc) + uploader := s3manager.NewUploaderWithClient(&s3Client) + up := s3manager.UploadInput{ + Bucket: s3Bucket, + Key: &key, + Body: bytes.NewReader(embeddingObj), + } + result, err := uploader.Upload(&up) + if err != nil { + log.Error(err) + return -1, stacktrace.Propagate(err, "") + } + + log.Infof("Uploaded to bucket %s", result.Location) + return len(embeddingObj), nil +} + +func (c *FileController) downloadObject(ctx context.Context, objectKey string, dc string) (ente.EmbeddingObject, error) { + var obj ente.EmbeddingObject + buff := &aws.WriteAtBuffer{} + bucket := c.S3Config.GetBucket(dc) + s3Client := c.S3Config.GetS3Client(dc) + downloader := s3manager.NewDownloaderWithClient(&s3Client) + _, err := downloader.DownloadWithContext(ctx, buff, &s3.GetObjectInput{ + Bucket: bucket, + Key: &objectKey, + }) + if err != nil { + return obj, err + } + err = json.Unmarshal(buff.Bytes(), &obj) + if err != nil { + return obj, stacktrace.Propagate(err, "unmarshal failed") + } + return obj, nil +} + +func (c *FileController) checkObjectExists(ctx context.Context, objectKey string, dc string) error { + s3Client := c.S3Config.GetS3Client(dc) + _, err := s3Client.HeadObject(&s3.HeadObjectInput{ + Bucket: c.S3Config.GetBucket(dc), + Key: &objectKey, + }) + if err != nil { + return err + } + return nil +} diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go new file mode 100644 index 0000000000..cb86174825 --- /dev/null +++ b/server/pkg/controller/filedata/controller.go @@ -0,0 +1,4 @@ +package filedata + +type Controller struct { +} diff --git a/server/pkg/controller/filedata/file_object.go b/server/pkg/controller/filedata/file_object.go new file mode 100644 index 0000000000..aa627fb82c --- /dev/null +++ b/server/pkg/controller/filedata/file_object.go @@ -0,0 +1,5 @@ +package filedata + +func (c *Controller) f() { + +} diff --git a/server/pkg/controller/preview/controller.go b/server/pkg/controller/preview/controller.go new file mode 100644 index 0000000000..e5ecaa71f0 --- /dev/null +++ b/server/pkg/controller/preview/controller.go @@ -0,0 +1,8 @@ +package preview + +type Controller struct { +} + +func NewController() *Controller { + return &Controller{} +} diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go new file mode 100644 index 0000000000..b06ddcfded --- /dev/null +++ b/server/pkg/repo/filedata/repository.go @@ -0,0 +1,172 @@ +package filedata + +import ( + "context" + "database/sql" + "github.com/ente-io/stacktrace" + "github.com/lib/pq" + "github.com/pkg/errors" +) + +// FileData represents the structure of the file_data table. +type FileData struct { + FileID int64 + UserID int64 + DataType string + Size int64 + LatestBucket string + ReplicatedBuckets []string + DeleteFromBuckets []string + PendingSync bool + IsDeleted bool + LastSyncTime int64 + CreatedAt int64 + UpdatedAt int64 +} + +// Repository defines the methods for inserting, updating, and retrieving file data. +type Repository struct { + DB *sql.DB +} + +// Insert inserts a new file_data record +func (r *Repository) Insert(ctx context.Context, data FileData) error { + query := ` + INSERT INTO file_data + (file_id, user_id, data_type, size, latest_bucket, replicated_buckets) + VALUES + ($1, $2, $3, $4, $5, $6) + ON CONFLICT (file_id, data_type) + DO UPDATE SET + size = $4, + latest_bucket = $5, + replicated_buckets = $6 ` + _, err := r.DB.ExecContext(ctx, query, + data.FileID, data.UserID, data.DataType, data.Size, data.LatestBucket, pq.Array(data.ReplicatedBuckets)) + if err != nil { + return stacktrace.Propagate(err, "failed to insert file data") + } + return nil +} + +// UpdateReplicatedBuckets updates the replicated_buckets for a given file and data type. +func (r *Repository) UpdateReplicatedBuckets(ctx context.Context, fileID int64, dataType string, newBuckets []string, previousUpdatedAt int64) error { + query := ` + UPDATE file_data + SET replicated_buckets = $1, updated_at = now_utc_micro_seconds() + WHERE file_id = $2 AND data_type = $3 AND updated_at = $4` + res, err := r.DB.ExecContext(ctx, query, pq.Array(newBuckets), fileID, dataType, previousUpdatedAt) + if err != nil { + return errors.Wrap(err, "failed to update replicated buckets") + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return errors.Wrap(err, "failed to check rows affected") + } + if rowsAffected == 0 { + return errors.New("no rows were updated, possible concurrent modification") + } + return nil +} + +// UpdateDeleteFromBuckets updates the delete_from_buckets for a given file and data type. +func (r *Repository) UpdateDeleteFromBuckets(ctx context.Context, fileID int64, dataType string, newBuckets []string, previousUpdatedAt int64) error { + query := ` + UPDATE file_data + SET delete_from_buckets = $1, updated_at = now_utc_micro_seconds() + WHERE file_id = $2 AND data_type = $3 AND updated_at = $4` + res, err := r.DB.ExecContext(ctx, query, pq.Array(newBuckets), fileID, dataType, previousUpdatedAt) + if err != nil { + return errors.Wrap(err, "failed to update delete from buckets") + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return errors.Wrap(err, "failed to check rows affected") + } + if rowsAffected == 0 { + return errors.New("no rows were updated, possible concurrent modification") + } + return nil +} + +// DeleteFileData deletes a file_data record by file_id and data_type if both replicated_buckets and delete_from_buckets are empty. +func (r *Repository) DeleteFileData(ctx context.Context, fileID int64, dataType string, previousUpdatedAt int64) error { + // First, check if both replicated_buckets and delete_from_buckets are empty. + var replicatedBuckets, deleteFromBuckets []string + query := `SELECT replicated_buckets, delete_from_buckets FROM file_data WHERE file_id = $1 AND data_type = $2` + err := r.DB.QueryRowContext(ctx, query, fileID, dataType).Scan(pq.Array(&replicatedBuckets), pq.Array(&deleteFromBuckets)) + if err != nil { + if err == sql.ErrNoRows { + return errors.New("no file data found for the given file_id and data_type") + } + return errors.Wrap(err, "failed to check buckets before deleting file data") + } + + if len(replicatedBuckets) > 0 || len(deleteFromBuckets) > 0 { + return errors.New("cannot delete file data with non-empty replicated_buckets or delete_from_buckets") + } + + // Proceed with deletion if both arrays are empty and updated_at matches. + deleteQuery := `DELETE FROM file_data WHERE file_id = $1 AND data_type = $2 AND updated_at = $3` + res, err := r.DB.ExecContext(ctx, deleteQuery, fileID, dataType, previousUpdatedAt) + if err != nil { + return errors.Wrap(err, "failed to delete file data") + } + + rowsAffected, err := res.RowsAffected() + if err != nil { + return errors.Wrap(err, "failed to check rows affected") + } + if rowsAffected == 0 { + return errors.New("no rows were deleted, possible concurrent modification") + } + return nil +} + +// GetFileData retrieves a single file_data record by file_id and data_type. +func (r *Repository) GetFileData(ctx context.Context, fileID int64, dataType string) (FileData, error) { + var data FileData + query := `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at + FROM file_data + WHERE file_id = $1 AND data_type = $2` + err := r.DB.QueryRowContext(ctx, query, fileID, dataType).Scan( + &data.FileID, &data.UserID, &data.DataType, &data.Size, &data.LatestBucket, pq.Array(&data.ReplicatedBuckets), pq.Array(&data.DeleteFromBuckets), &data.PendingSync, &data.IsDeleted, &data.LastSyncTime, &data.CreatedAt, &data.UpdatedAt, + ) + if err != nil { + if err == sql.ErrNoRows { + return FileData{}, errors.Wrap(err, "no file data found") + } + return FileData{}, errors.Wrap(err, "failed to retrieve file data") + } + return data, nil +} + +// ListFileData retrieves all file_data records for a given user_id. +func (r *Repository) ListFileData(ctx context.Context, userID int64) ([]FileData, error) { + query := `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at + FROM file_data + WHERE user_id = $1` + rows, err := r.DB.QueryContext(ctx, query, userID) + if err != nil { + return nil, errors.Wrap(err, "failed to list file data") + } + defer rows.Close() + + var fileDataList []FileData + for rows.Next() { + var data FileData + err := rows.Scan( + &data.FileID, &data.UserID, &data.DataType, &data.Size, &data.LatestBucket, pq.Array(&data.ReplicatedBuckets), pq.Array(&data.DeleteFromBuckets), &data.PendingSync, &data.IsDeleted, &data.LastSyncTime, &data.CreatedAt, &data.UpdatedAt, + ) + if err != nil { + return nil, errors.Wrap(err, "failed to scan file data row") + } + fileDataList = append(fileDataList, data) + } + if err = rows.Err(); err != nil { + return nil, errors.Wrap(err, "error iterating file data rows") + } + return fileDataList, nil +} From 8f8f5d2f51e1a6d2e42893369ca083c7b096e5e2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:47:44 +0530 Subject: [PATCH 005/211] [server] Avoid redirect --- server/pkg/api/file.go | 4 +++- server/pkg/controller/file_preview.go | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go index 94760049af..d2d12b91f7 100644 --- a/server/pkg/api/file.go +++ b/server/pkg/api/file.go @@ -138,7 +138,9 @@ func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) { handler.Error(c, stacktrace.Propagate(err, "")) return } - c.Redirect(http.StatusTemporaryRedirect, url) + c.JSON(http.StatusOK, gin.H{ + "url": url, + }) } func (h *FileHandler) ReportVideoPlayList(c *gin.Context) { diff --git a/server/pkg/controller/file_preview.go b/server/pkg/controller/file_preview.go index 1aa3bfb1dc..d998267319 100644 --- a/server/pkg/controller/file_preview.go +++ b/server/pkg/controller/file_preview.go @@ -46,6 +46,11 @@ func (c *FileController) GetPreviewUrl(ctx context.Context, userID int64, fileID return "", err } objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video" + // check if playlist exists + err = c.checkObjectExists(ctx, objectKey+"_playlist.m3u8", c.S3Config.GetDerivedStorageDataCenter()) + if err != nil { + return "", stacktrace.Propagate(ente.NewBadRequestWithMessage("Video playlist does not exist"), fmt.Sprintf("objectKey: %s", objectKey)) + } s3Client := c.S3Config.GetDerivedStorageS3Client() r, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{ Bucket: c.S3Config.GetDerivedStorageBucket(), From dc143bbaaf46d4f21cd4931d19840f33309cbd2e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 29 Jul 2024 15:48:22 +0530 Subject: [PATCH 006/211] [server] Minor fix --- server/cmd/museum/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index 3fb5359289..fda21bc659 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -412,7 +412,7 @@ func main() { privateAPI.GET("/files/file-data/playlist/:fileID", fileHandler.GetVideoPlaylist) privateAPI.POST("/files/file-data/playlist", fileHandler.ReportVideoPlayList) privateAPI.GET("/files/file-data/preview/upload-url/:fileID", fileHandler.GetVideoUploadURL) - privateAPI.GET("/files/file-data/preview/:fileID", fileHandler.GetVideoUploadURL) + privateAPI.GET("/files/file-data/preview/:fileID", fileHandler.GetVideoPreviewUrl) privateAPI.POST("/files", fileHandler.CreateOrUpdate) privateAPI.POST("/files/copy", fileHandler.CopyFiles) From 3a962cfe522380e66768a6b68987afb946ded288 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 30 Jul 2024 10:30:57 +0530 Subject: [PATCH 007/211] [server] Extend ObjectType instead of creating new enum --- server/ente/file.go | 7 ++- server/ente/fileobjects/type.go | 45 ------------------- .../migrations/89_derived_data_table.up.sql | 5 +-- 3 files changed, 6 insertions(+), 51 deletions(-) delete mode 100644 server/ente/fileobjects/type.go diff --git a/server/ente/file.go b/server/ente/file.go index a0e67c71cf..8a554b6b1b 100644 --- a/server/ente/file.go +++ b/server/ente/file.go @@ -153,8 +153,11 @@ type MultipartUploadURLs struct { type ObjectType string const ( - FILE ObjectType = "file" - THUMBNAIL ObjectType = "thumbnail" + FILE ObjectType = "file" + THUMBNAIL ObjectType = "thumbnail" + PreviewImage ObjectType = "img_preview" + PreviewVideo ObjectType = "vid_preview" + DerivedMeta ObjectType = "derivedMeta" ) // S3ObjectKey represents the s3 object key and corresponding fileID for it diff --git a/server/ente/fileobjects/type.go b/server/ente/fileobjects/type.go deleted file mode 100644 index 8c5ce65919..0000000000 --- a/server/ente/fileobjects/type.go +++ /dev/null @@ -1,45 +0,0 @@ -package fileobjects - -import ( - "database/sql/driver" - "errors" - "fmt" -) - -type Type string - -const ( - OriginalFile Type = "file" - OriginalThumbnail Type = "thumb" - PreviewImage Type = "previewImage" - PreviewVideo Type = "previewVideo" - Derived Type = "derived" -) - -func (ft Type) IsValid() bool { - switch ft { - case OriginalFile, OriginalThumbnail, PreviewImage, PreviewVideo, Derived: - return true - } - return false -} - -func (ft *Type) Scan(value interface{}) error { - strValue, ok := value.(string) - if !ok { - return errors.New("type should be a string") - } - - *ft = Type(strValue) - if !ft.IsValid() { - return fmt.Errorf("invalid FileType value: %s", strValue) - } - return nil -} - -func (ft Type) Value() (driver.Value, error) { - if !ft.IsValid() { - return nil, fmt.Errorf("invalid FileType value: %s", ft) - } - return string(ft), nil -} diff --git a/server/migrations/89_derived_data_table.up.sql b/server/migrations/89_derived_data_table.up.sql index 548af5f12a..549d3ca06c 100644 --- a/server/migrations/89_derived_data_table.up.sql +++ b/server/migrations/89_derived_data_table.up.sql @@ -1,11 +1,8 @@ --- Create the data_type enum -CREATE TYPE file_data_type AS ENUM ('img_jpg_preview', 'vid_hls_preview', 'derived'); - -- Create the derived table CREATE TABLE file_data ( file_id BIGINT NOT NULL, user_id BIGINT NOT NULL, - data_type file_data_type NOT NULL, + data_type OBJECT_TYPE NOT NULL, size BIGINT NOT NULL, latest_bucket s3region NOT NULL, replicated_buckets s3region[] NOT NULL, From b9b22fa4dc6d22c630fd45a038a7e0c736657b1f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:04:51 +0530 Subject: [PATCH 008/211] [server] Add DB triggers for validating row sanity --- .../migrations/89_derived_data_table.down.sql | 3 +- .../migrations/89_derived_data_table.up.sql | 66 +++++++++++++------ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/server/migrations/89_derived_data_table.down.sql b/server/migrations/89_derived_data_table.down.sql index a6f4e7e3d0..1d39d0663f 100644 --- a/server/migrations/89_derived_data_table.down.sql +++ b/server/migrations/89_derived_data_table.down.sql @@ -1,6 +1,5 @@ -DROP INDEX IF EXISTS idx_file_data_user_id_data_type; -DROP INDEX IF EXISTS idx_file_data_user_id_updated_at; +DROP INDEX IF EXISTS idx_file_data_user_type_deleted; DROP TABLE IF EXISTS file_data; diff --git a/server/migrations/89_derived_data_table.up.sql b/server/migrations/89_derived_data_table.up.sql index 549d3ca06c..042f46c564 100644 --- a/server/migrations/89_derived_data_table.up.sql +++ b/server/migrations/89_derived_data_table.up.sql @@ -1,24 +1,52 @@ -- Create the derived table -CREATE TABLE file_data ( - file_id BIGINT NOT NULL, - user_id BIGINT NOT NULL, - data_type OBJECT_TYPE NOT NULL, - size BIGINT NOT NULL, - latest_bucket s3region NOT NULL, - replicated_buckets s3region[] NOT NULL, --- following field contains list of buckets from where we need to delete the data as the given data_type will not longer be persisted in that dc - delete_from_buckets s3region[] NOT NULL DEFAULT '{}', - pending_sync BOOLEAN NOT NULL DEFAULT false, - is_deleted BOOLEAN NOT NULL DEFAULT false, - last_sync_time BIGINT NOT NULL DEFAULT 0, - created_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), - updated_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), - PRIMARY KEY (file_id, data_type) +CREATE TABLE file_data +( + file_id BIGINT NOT NULL, + user_id BIGINT NOT NULL, + data_type OBJECT_TYPE NOT NULL, + size BIGINT NOT NULL, + latest_bucket s3region NOT NULL, + replicated_buckets s3region[] NOT NULL DEFAULT '{}', +-- following field contains list of buckets from where we need to delete the data as the given data_type will not longer be persisted in that dc + delete_from_buckets s3region[] NOT NULL DEFAULT '{}', + pending_sync BOOLEAN NOT NULL DEFAULT false, + is_deleted BOOLEAN NOT NULL DEFAULT false, + last_sync_time BIGINT NOT NULL DEFAULT 0, + created_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), + updated_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), + PRIMARY KEY (file_id, data_type) ); --- Add index for user_id and data_type -CREATE INDEX idx_file_data_user_id_data_type ON file_data (user_id, data_type); +-- Add index for user_id and data_type for efficient querying +CREATE INDEX idx_file_data_user_type_deleted ON file_data (user_id, data_type, is_deleted) INCLUDE (file_id, size); + +CREATE OR REPLACE FUNCTION ensure_no_common_entries() + RETURNS TRIGGER AS $$ +BEGIN + -- Check for common entries between latest_bucket and replicated_buckets + IF NEW.latest_bucket = ANY(NEW.replicated_buckets) THEN + RAISE EXCEPTION 'latest_bucket and replicated_buckets have common entries'; + END IF; + + -- Check for common entries between latest_bucket and delete_from_buckets + IF NEW.latest_bucket = ANY(NEW.delete_from_buckets) THEN + RAISE EXCEPTION 'latest_bucket and delete_from_buckets have common entries'; + END IF; + + -- Check for common entries between replicated_buckets and delete_from_buckets + IF EXISTS ( + SELECT 1 FROM unnest(NEW.replicated_buckets) AS rb + WHERE rb = ANY(NEW.delete_from_buckets) + ) THEN + RAISE EXCEPTION 'replicated_buckets and delete_from_buckets have common entries'; + END IF; + + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER check_no_common_entries + BEFORE INSERT OR UPDATE ON file_data + FOR EACH ROW EXECUTE FUNCTION ensure_no_common_entries(); --- Add index for user_id and updated_at for efficient querying -CREATE INDEX idx_file_data_user_id_updated_at ON file_data (user_id, updated_at); \ No newline at end of file From 5bd75a8567b15e2557710b88c406b02e978b083f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:42:31 +0530 Subject: [PATCH 009/211] [server] Add req/res model files --- server/ente/filedata/filedata.go | 67 ++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 server/ente/filedata/filedata.go diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go new file mode 100644 index 0000000000..27c0386e5e --- /dev/null +++ b/server/ente/filedata/filedata.go @@ -0,0 +1,67 @@ +package filedata + +import ( + "fmt" + "github.com/ente-io/museum/ente" +) + +type PutFileDataRequest struct { + FileID int64 `json:"fileID" binding:"required"` + Type ente.ObjectType `json:"type" binding:"required"` + EncryptedData *string `json:"encryptedData"` + DecryptionHeader *string `json:"decryptionHeader"` + // ObjectKey is the key of the object in the S3 bucket. This is needed while putting the object in the S3 bucket. + ObjectKey *string `json:"objectKey"` + // size of the object that is being uploaded. This helps in checking the size of the object that is being uploaded. + Size *int64 `json:"size" ` +} + +func (r PutFileDataRequest) Validate() error { + switch r.Type { + case ente.PreviewVideo: + if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { + // the video playlist is uploaded as part of encrypted data and decryption header + return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for preview video") + } + if r.Size == nil || r.ObjectKey == nil { + return ente.NewBadRequestWithMessage("size and objectKey are required for preview video") + } + case ente.PreviewImage: + if r.Size == nil || r.ObjectKey == nil { + return ente.NewBadRequestWithMessage("size and objectKey are required for preview image") + } + case ente.DerivedMeta: + if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { + return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for derived meta") + } + default: + return ente.NewBadRequestWithMessage(fmt.Sprintf("invalid object type %s", r.Type)) + } + return nil +} + +// GetFilesData should only be used for getting the preview video playlist and derived metadata. +type GetFilesData struct { + FileIDs []int64 `form:"fileIDs" binding:"required"` + Type ente.ObjectType `json:"type" binding:"required"` +} + +func (g *GetFilesData) Validate() error { + if g.Type != ente.PreviewVideo && g.Type != ente.DerivedMeta { + return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) + } + return nil +} + +type Entity struct { + FileID int64 `json:"fileID"` + Type ente.ObjectType `json:"type"` + EncryptedData string `json:"encryptedData"` + DecryptionHeader string `json:"decryptionHeader"` +} + +type GetFilesDataResponse struct { + Data []Entity `json:"data"` + PendingIndexFileIDs []int64 `json:"pendingIndexFileIDs"` + ErrFileIDs []int64 `json:"errFileIDs"` +} From a67bc6aee7c8fb0a9110c2ecc7e03671c6fe09a6 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:24:25 +0530 Subject: [PATCH 010/211] [server] Add bucket5 --- server/pkg/utils/s3config/s3config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index a562e51815..204ee4e06c 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -74,6 +74,7 @@ var ( dcWasabiEuropeCentral_v3 string = "wasabi-eu-central-2-v3" dcSCWEuropeFrance_v3 string = "scw-eu-fr-v3" dcWasabiEuropeCentralDerived string = "wasabi-eu-central-2-derived" + bucket5 string = "b5" ) // Number of days that the wasabi bucket is configured to retain objects. @@ -89,9 +90,9 @@ func NewS3Config() *S3Config { } func (config *S3Config) initialize() { - dcs := [6]string{ + dcs := [7]string{ dcB2EuropeCentral, dcSCWEuropeFranceLockedDeprecated, dcWasabiEuropeCentralDeprecated, - dcWasabiEuropeCentral_v3, dcSCWEuropeFrance_v3, dcWasabiEuropeCentralDerived} + dcWasabiEuropeCentral_v3, dcSCWEuropeFrance_v3, dcWasabiEuropeCentralDerived, bucket5} config.hotDC = dcB2EuropeCentral config.secondaryHotDC = dcWasabiEuropeCentral_v3 From bfe5632477e5edb27ef87b43ff758e1c5d75a427 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 1 Aug 2024 13:24:46 +0530 Subject: [PATCH 011/211] [server] Support for inserting and fetching s3 metadata --- server/ente/filedata/filedata.go | 76 ++--- server/ente/filedata/path.go | 42 +++ server/ente/filedata/putfiledata.go | 52 ++++ server/pkg/controller/filedata/controller.go | 275 +++++++++++++++++++ server/pkg/controller/filedata/s3.go | 52 ++++ server/pkg/repo/filedata/repository.go | 178 +++--------- 6 files changed, 500 insertions(+), 175 deletions(-) create mode 100644 server/ente/filedata/path.go create mode 100644 server/ente/filedata/putfiledata.go create mode 100644 server/pkg/controller/filedata/s3.go diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go index 27c0386e5e..807f2b5e6b 100644 --- a/server/ente/filedata/filedata.go +++ b/server/ente/filedata/filedata.go @@ -5,41 +5,6 @@ import ( "github.com/ente-io/museum/ente" ) -type PutFileDataRequest struct { - FileID int64 `json:"fileID" binding:"required"` - Type ente.ObjectType `json:"type" binding:"required"` - EncryptedData *string `json:"encryptedData"` - DecryptionHeader *string `json:"decryptionHeader"` - // ObjectKey is the key of the object in the S3 bucket. This is needed while putting the object in the S3 bucket. - ObjectKey *string `json:"objectKey"` - // size of the object that is being uploaded. This helps in checking the size of the object that is being uploaded. - Size *int64 `json:"size" ` -} - -func (r PutFileDataRequest) Validate() error { - switch r.Type { - case ente.PreviewVideo: - if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { - // the video playlist is uploaded as part of encrypted data and decryption header - return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for preview video") - } - if r.Size == nil || r.ObjectKey == nil { - return ente.NewBadRequestWithMessage("size and objectKey are required for preview video") - } - case ente.PreviewImage: - if r.Size == nil || r.ObjectKey == nil { - return ente.NewBadRequestWithMessage("size and objectKey are required for preview image") - } - case ente.DerivedMeta: - if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { - return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for derived meta") - } - default: - return ente.NewBadRequestWithMessage(fmt.Sprintf("invalid object type %s", r.Type)) - } - return nil -} - // GetFilesData should only be used for getting the preview video playlist and derived metadata. type GetFilesData struct { FileIDs []int64 `form:"fileIDs" binding:"required"` @@ -50,6 +15,12 @@ func (g *GetFilesData) Validate() error { if g.Type != ente.PreviewVideo && g.Type != ente.DerivedMeta { return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) } + if len(g.FileIDs) == 0 { + return ente.NewBadRequestWithMessage("fileIDs are required") + } + if len(g.FileIDs) > 200 { + return ente.NewBadRequestWithMessage("fileIDs should be less than or equal to 200") + } return nil } @@ -60,8 +31,43 @@ type Entity struct { DecryptionHeader string `json:"decryptionHeader"` } +// Row represents the data that is stored in the file_data table. +type Row struct { + FileID int64 + UserID int64 + Type ente.ObjectType + Size int64 + LatestBucket string + ReplicatedBuckets []string + DeleteFromBuckets []string + PendingSync bool + IsDeleted bool + LastSyncTime int64 + CreatedAt int64 + UpdatedAt int64 +} + +func (r Row) S3FileMetadataObjectKey() string { + if r.Type == ente.DerivedMeta { + return derivedMetaPath(r.FileID, r.UserID) + } + if r.Type == ente.PreviewVideo { + return previewVideoPlaylist(r.FileID, r.UserID) + } + panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) +} + type GetFilesDataResponse struct { Data []Entity `json:"data"` PendingIndexFileIDs []int64 `json:"pendingIndexFileIDs"` ErrFileIDs []int64 `json:"errFileIDs"` } + +// S3FileMetadata stuck represents the metadata that is stored in the S3 bucket for non-file type metadata +// that is stored in the S3 bucket. +type S3FileMetadata struct { + Version int `json:"v"` + EncryptedData string `json:"encryptedData"` + DecryptionHeader string `json:"header"` + Client string `json:"client"` +} diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.go new file mode 100644 index 0000000000..c656fce164 --- /dev/null +++ b/server/ente/filedata/path.go @@ -0,0 +1,42 @@ +package filedata + +import ( + "fmt" + "github.com/ente-io/museum/ente" +) + +// basePrefix returns the base prefix for all objects related to a file. To check if the file data is deleted, +// ensure that there's no file in the S3 bucket with this prefix. +func basePrefix(fileID int64, ownerID int64) string { + return fmt.Sprintf("%d/file-data/%d/", ownerID, fileID) +} + +func allObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string { + switch oType { + case ente.PreviewVideo: + return []string{previewVideoPath(fileID, ownerID), previewVideoPlaylist(fileID, ownerID)} + case ente.DerivedMeta: + return []string{derivedMetaPath(fileID, ownerID)} + case ente.PreviewImage: + return []string{previewImagePath(fileID, ownerID)} + default: + // throw panic saying current object type is not supported + panic(fmt.Sprintf("object type %s is not supported", oType)) + } +} + +func previewVideoPath(fileID int64, ownerID int64) string { + return fmt.Sprintf("%s%s", basePrefix(fileID, ownerID), string(ente.PreviewVideo)) +} + +func previewVideoPlaylist(fileID int64, ownerID int64) string { + return fmt.Sprintf("%s%s", previewVideoPath(fileID, ownerID), "_playlist.m3u8") +} + +func previewImagePath(fileID int64, ownerID int64) string { + return fmt.Sprintf("%s%s", basePrefix(fileID, ownerID), string(ente.PreviewImage)) +} + +func derivedMetaPath(fileID int64, ownerID int64) string { + return fmt.Sprintf("%s%s", basePrefix(fileID, ownerID), string(ente.DerivedMeta)) +} diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go new file mode 100644 index 0000000000..6a5c5b0630 --- /dev/null +++ b/server/ente/filedata/putfiledata.go @@ -0,0 +1,52 @@ +package filedata + +import ( + "fmt" + "github.com/ente-io/museum/ente" +) + +type PutFileDataRequest struct { + FileID int64 `json:"fileID" binding:"required"` + Type ente.ObjectType `json:"type" binding:"required"` + EncryptedData *string `json:"encryptedData,omitempty"` + DecryptionHeader *string `json:"decryptionHeader,omitempty"` + // ObjectKey is the key of the object in the S3 bucket. This is needed while putting the object in the S3 bucket. + ObjectKey *string `json:"objectKey,omitempty"` + // size of the object that is being uploaded. This helps in checking the size of the object that is being uploaded. + Size *int64 `json:"size,omitempty"` + Version *int `json:"version,omitempty"` +} + +func (r PutFileDataRequest) Validate() error { + switch r.Type { + case ente.PreviewVideo: + if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { + // the video playlist is uploaded as part of encrypted data and decryption header + return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for preview video") + } + if r.Size == nil || r.ObjectKey == nil { + return ente.NewBadRequestWithMessage("size and objectKey are required for preview video") + } + case ente.PreviewImage: + if r.Size == nil || r.ObjectKey == nil { + return ente.NewBadRequestWithMessage("size and objectKey are required for preview image") + } + case ente.DerivedMeta: + if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { + return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for derived meta") + } + default: + return ente.NewBadRequestWithMessage(fmt.Sprintf("invalid object type %s", r.Type)) + } + return nil +} + +func (r PutFileDataRequest) S3FileMetadataObjectKey(ownerID int64) string { + if r.Type == ente.DerivedMeta { + return derivedMetaPath(r.FileID, ownerID) + } + if r.Type == ente.PreviewVideo { + return previewVideoPlaylist(r.FileID, ownerID) + } + panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) +} diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index cb86174825..d07f90029e 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -1,4 +1,279 @@ package filedata +import ( + "context" + "errors" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/ente-io/museum/ente" + fileData "github.com/ente-io/museum/ente/filedata" + "github.com/ente-io/museum/pkg/controller" + "github.com/ente-io/museum/pkg/controller/access" + "github.com/ente-io/museum/pkg/repo" + fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata" + "github.com/ente-io/museum/pkg/utils/array" + "github.com/ente-io/museum/pkg/utils/auth" + "github.com/ente-io/museum/pkg/utils/network" + "github.com/ente-io/museum/pkg/utils/s3config" + "github.com/ente-io/stacktrace" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + "sync" + gTime "time" +) + +const ( + embeddingFetchTimeout = 10 * gTime.Second +) + +// _fetchConfig is the configuration for the fetching objects from S3 +type _fetchConfig struct { + RetryCount int + InitialTimeout gTime.Duration + MaxTimeout gTime.Duration +} + +var _defaultFetchConfig = _fetchConfig{RetryCount: 3, InitialTimeout: 10 * gTime.Second, MaxTimeout: 30 * gTime.Second} +var globalFileFetchSemaphore = make(chan struct{}, 400) + +type bulkS3MetaFetchResult struct { + s3MetaObject fileData.S3FileMetadata + dbEntry fileData.Row + err error +} + type Controller struct { + Repo *fileDataRepo.Repository + AccessCtrl access.Controller + ObjectCleanupController *controller.ObjectCleanupController + S3Config *s3config.S3Config + QueueRepo *repo.QueueRepository + TaskLockingRepo *repo.TaskLockRepository + FileRepo *repo.FileRepository + CollectionRepo *repo.CollectionRepository + HostName string + cleanupCronRunning bool + derivedStorageDataCenter string + downloadManagerCache map[string]*s3manager.Downloader +} + +func New(repo *fileDataRepo.Repository, + accessCtrl access.Controller, + objectCleanupController *controller.ObjectCleanupController, + s3Config *s3config.S3Config, + queueRepo *repo.QueueRepository, + taskLockingRepo *repo.TaskLockRepository, + fileRepo *repo.FileRepository, + collectionRepo *repo.CollectionRepository, + hostName string) *Controller { + embeddingDcs := []string{s3Config.GetHotBackblazeDC(), s3Config.GetHotWasabiDC(), s3Config.GetWasabiDerivedDC(), s3Config.GetDerivedStorageDataCenter()} + cache := make(map[string]*s3manager.Downloader, len(embeddingDcs)) + for i := range embeddingDcs { + s3Client := s3Config.GetS3Client(embeddingDcs[i]) + cache[embeddingDcs[i]] = s3manager.NewDownloaderWithClient(&s3Client) + } + return &Controller{ + Repo: repo, + AccessCtrl: accessCtrl, + ObjectCleanupController: objectCleanupController, + S3Config: s3Config, + QueueRepo: queueRepo, + TaskLockingRepo: taskLockingRepo, + FileRepo: fileRepo, + CollectionRepo: collectionRepo, + HostName: hostName, + derivedStorageDataCenter: s3Config.GetDerivedStorageDataCenter(), + downloadManagerCache: cache, + } +} + +func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataRequest) error { + if err := req.Validate(); err != nil { + return stacktrace.Propagate(err, "validation failed") + } + userID := auth.GetUserID(ctx.Request.Header) + err := c._validateInsertPermission(ctx, req.FileID, userID) + if err != nil { + return stacktrace.Propagate(err, "") + } + fileOwnerID := userID + objectKey := req.S3FileMetadataObjectKey(fileOwnerID) + obj := fileData.S3FileMetadata{ + Version: *req.Version, + EncryptedData: *req.EncryptedData, + DecryptionHeader: *req.DecryptionHeader, + Client: network.GetClientInfo(ctx), + } + size, uploadErr := c.uploadObject(obj, objectKey, c.derivedStorageDataCenter) + if uploadErr != nil { + log.Error(uploadErr) + return stacktrace.Propagate(uploadErr, "") + } + row := fileData.Row{ + FileID: req.FileID, + Type: req.Type, + UserID: fileOwnerID, + Size: size, + LatestBucket: c.derivedStorageDataCenter, + } + err = c.Repo.InsertOrUpdate(ctx, row) + if err != nil { + return stacktrace.Propagate(err, "") + } + return nil +} + +func (c *Controller) GetFilesData(ctx *gin.Context, req fileData.GetFilesData) (*fileData.GetFilesDataResponse, error) { + userID := auth.GetUserID(ctx.Request.Header) + if err := c._validateGetFilesData(ctx, userID, req); err != nil { + return nil, stacktrace.Propagate(err, "") + } + + doRows, err := c.Repo.GetFilesData(ctx, req.Type, req.FileIDs) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + + activeRows := make([]fileData.Row, 0) + dbFileIds := make([]int64, 0) + errFileIds := make([]int64, 0) + for i := range doRows { + dbFileIds = append(dbFileIds, doRows[i].FileID) + if doRows[i].IsDeleted == false { + activeRows = append(activeRows, doRows[i]) + } + } + pendingIndexFileIds := array.FindMissingElementsInSecondList(req.FileIDs, dbFileIds) + // Fetch missing doRows in parallel + s3MetaFetchResults, err := c.getS3FileMetadataParallel(activeRows, c.derivedStorageDataCenter) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + fetchedEmbeddings := make([]fileData.Entity, 0) + + // Populate missing data in doRows from fetched objects + for _, obj := range s3MetaFetchResults { + if obj.err != nil { + errFileIds = append(errFileIds, obj.dbEntry.FileID) + } else { + fetchedEmbeddings = append(fetchedEmbeddings, fileData.Entity{ + FileID: obj.dbEntry.FileID, + Type: obj.dbEntry.Type, + EncryptedData: obj.s3MetaObject.EncryptedData, + DecryptionHeader: obj.s3MetaObject.DecryptionHeader, + }) + } + } + + return &fileData.GetFilesDataResponse{ + Data: fetchedEmbeddings, + PendingIndexFileIDs: pendingIndexFileIds, + ErrFileIDs: errFileIds, + }, nil +} + +func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row, dc string) ([]bulkS3MetaFetchResult, error) { + var wg sync.WaitGroup + embeddingObjects := make([]bulkS3MetaFetchResult, len(dbRows)) + for i, _ := range dbRows { + dbRow := dbRows[i] + wg.Add(1) + globalFileFetchSemaphore <- struct{}{} // Acquire from global semaphore + go func(i int, row fileData.Row) { + defer wg.Done() + defer func() { <-globalFileFetchSemaphore }() // Release back to global semaphore + s3FileMetadata, err := c.fetchS3FileMetadata(context.Background(), row, dc) + if err != nil { + log.Error("error fetching embedding object: "+row.S3FileMetadataObjectKey(), err) + embeddingObjects[i] = bulkS3MetaFetchResult{ + err: err, + dbEntry: row, + } + + } else { + embeddingObjects[i] = bulkS3MetaFetchResult{ + s3MetaObject: *s3FileMetadata, + dbEntry: dbRow, + } + } + }(i, dbRow) + } + wg.Wait() + return embeddingObjects, nil +} + +func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, dc string) (*fileData.S3FileMetadata, error) { + opt := _defaultFetchConfig + objectKey := row.S3FileMetadataObjectKey() + ctxLogger := log.WithField("objectKey", objectKey).WithField("dc", dc) + totalAttempts := opt.RetryCount + 1 + timeout := opt.InitialTimeout + for i := 0; i < totalAttempts; i++ { + if i > 0 { + timeout = timeout * 2 + if timeout > opt.MaxTimeout { + timeout = opt.MaxTimeout + } + } + fetchCtx, cancel := context.WithTimeout(ctx, timeout) + select { + case <-ctx.Done(): + cancel() + return nil, stacktrace.Propagate(ctx.Err(), "") + default: + obj, err := c.downloadObject(fetchCtx, objectKey, dc) + cancel() // Ensure cancel is called to release resources + if err == nil { + if i > 0 { + ctxLogger.Infof("Fetched object after %d attempts", i) + } + return &obj, nil + } + // Check if the error is due to context timeout or cancellation + if err == nil && fetchCtx.Err() != nil { + ctxLogger.Error("Fetch timed out or cancelled: ", fetchCtx.Err()) + } else { + // check if the error is due to object not found + if s3Err, ok := err.(awserr.RequestFailure); ok { + if s3Err.Code() == s3.ErrCodeNoSuchKey { + return nil, stacktrace.Propagate(errors.New("object not found"), "") + } + } + ctxLogger.Error("Failed to fetch object: ", err) + } + } + } + return nil, stacktrace.Propagate(errors.New("failed to fetch object"), "") +} + +func (c *Controller) _validateGetFilesData(ctx *gin.Context, userID int64, req fileData.GetFilesData) error { + if err := req.Validate(); err != nil { + return stacktrace.Propagate(err, "validation failed") + } + if err := c.AccessCtrl.VerifyFileOwnership(ctx, &access.VerifyFileOwnershipParams{ + ActorUserId: userID, + FileIDs: req.FileIDs, + }); err != nil { + return stacktrace.Propagate(err, "User does not own some file(s)") + } + return nil +} + +func (c *Controller) _validateInsertPermission(ctx *gin.Context, fileID int64, actorID int64) error { + err := c.AccessCtrl.VerifyFileOwnership(ctx, &access.VerifyFileOwnershipParams{ + ActorUserId: actorID, + FileIDs: []int64{fileID}, + }) + if err != nil { + return stacktrace.Propagate(err, "User does not own file") + } + count, err := c.CollectionRepo.GetCollectionCount(fileID) + if err != nil { + return stacktrace.Propagate(err, "") + } + if count < 1 { + return stacktrace.Propagate(ente.ErrNotFound, "") + } + return nil } diff --git a/server/pkg/controller/filedata/s3.go b/server/pkg/controller/filedata/s3.go new file mode 100644 index 0000000000..f8a55052ec --- /dev/null +++ b/server/pkg/controller/filedata/s3.go @@ -0,0 +1,52 @@ +package filedata + +import ( + "bytes" + "context" + "encoding/json" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + fileData "github.com/ente-io/museum/ente/filedata" + "github.com/ente-io/stacktrace" + log "github.com/sirupsen/logrus" +) + +func (c *Controller) downloadObject(ctx context.Context, objectKey string, dc string) (fileData.S3FileMetadata, error) { + var obj fileData.S3FileMetadata + buff := &aws.WriteAtBuffer{} + bucket := c.S3Config.GetBucket(dc) + downloader := c.downloadManagerCache[dc] + _, err := downloader.DownloadWithContext(ctx, buff, &s3.GetObjectInput{ + Bucket: bucket, + Key: &objectKey, + }) + if err != nil { + return obj, err + } + err = json.Unmarshal(buff.Bytes(), &obj) + if err != nil { + return obj, stacktrace.Propagate(err, "unmarshal failed") + } + return obj, nil +} + +// uploadObject uploads the embedding object to the object store and returns the object size +func (c *Controller) uploadObject(obj fileData.S3FileMetadata, objectKey string, dc string) (int64, error) { + embeddingObj, _ := json.Marshal(obj) + s3Client := c.S3Config.GetS3Client(dc) + s3Bucket := c.S3Config.GetBucket(dc) + uploader := s3manager.NewUploaderWithClient(&s3Client) + up := s3manager.UploadInput{ + Bucket: s3Bucket, + Key: &objectKey, + Body: bytes.NewReader(embeddingObj), + } + result, err := uploader.Upload(&up) + if err != nil { + log.Error(err) + return -1, stacktrace.Propagate(err, "") + } + log.Infof("Uploaded to bucket %s", result.Location) + return int64(len(embeddingObj)), nil +} diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go index b06ddcfded..02d7ccb57b 100644 --- a/server/pkg/repo/filedata/repository.go +++ b/server/pkg/repo/filedata/repository.go @@ -3,170 +3,68 @@ package filedata import ( "context" "database/sql" + "github.com/ente-io/museum/ente" + "github.com/ente-io/museum/ente/filedata" "github.com/ente-io/stacktrace" "github.com/lib/pq" - "github.com/pkg/errors" ) -// FileData represents the structure of the file_data table. -type FileData struct { - FileID int64 - UserID int64 - DataType string - Size int64 - LatestBucket string - ReplicatedBuckets []string - DeleteFromBuckets []string - PendingSync bool - IsDeleted bool - LastSyncTime int64 - CreatedAt int64 - UpdatedAt int64 -} - // Repository defines the methods for inserting, updating, and retrieving file data. type Repository struct { DB *sql.DB } -// Insert inserts a new file_data record -func (r *Repository) Insert(ctx context.Context, data FileData) error { +func (r *Repository) InsertOrUpdate(ctx context.Context, data filedata.Row) error { query := ` - INSERT INTO file_data - (file_id, user_id, data_type, size, latest_bucket, replicated_buckets) - VALUES - ($1, $2, $3, $4, $5, $6) - ON CONFLICT (file_id, data_type) - DO UPDATE SET - size = $4, - latest_bucket = $5, - replicated_buckets = $6 ` + INSERT INTO file_data + (file_id, user_id, data_type, size, latest_bucket) + VALUES + ($1, $2, $3, $4, $5) + ON CONFLICT (file_id, data_type) + DO UPDATE SET + size = EXCLUDED.size, + delete_from_buckets = array( + SELECT DISTINCT elem FROM unnest( + array_append( + array_cat(file_data.replicated_buckets, file_data.delete_from_buckets), + CASE WHEN file_data.latest_bucket != EXCLUDED.latest_bucket THEN file_data.latest_bucket END + ) + ) AS elem + WHERE elem IS NOT NULL AND elem != EXCLUDED.latest_bucket + ), + replicated_buckets = ARRAY[]::s3region[], + latest_bucket = EXCLUDED.latest_bucket, + updated_at = now_utc_micro_seconds() + WHERE file_data.is_deleted = false` _, err := r.DB.ExecContext(ctx, query, - data.FileID, data.UserID, data.DataType, data.Size, data.LatestBucket, pq.Array(data.ReplicatedBuckets)) + data.FileID, data.UserID, string(data.Type), data.Size, data.LatestBucket) if err != nil { return stacktrace.Propagate(err, "failed to insert file data") } return nil } -// UpdateReplicatedBuckets updates the replicated_buckets for a given file and data type. -func (r *Repository) UpdateReplicatedBuckets(ctx context.Context, fileID int64, dataType string, newBuckets []string, previousUpdatedAt int64) error { - query := ` - UPDATE file_data - SET replicated_buckets = $1, updated_at = now_utc_micro_seconds() - WHERE file_id = $2 AND data_type = $3 AND updated_at = $4` - res, err := r.DB.ExecContext(ctx, query, pq.Array(newBuckets), fileID, dataType, previousUpdatedAt) - if err != nil { - return errors.Wrap(err, "failed to update replicated buckets") - } - - rowsAffected, err := res.RowsAffected() - if err != nil { - return errors.Wrap(err, "failed to check rows affected") - } - if rowsAffected == 0 { - return errors.New("no rows were updated, possible concurrent modification") - } - return nil -} - -// UpdateDeleteFromBuckets updates the delete_from_buckets for a given file and data type. -func (r *Repository) UpdateDeleteFromBuckets(ctx context.Context, fileID int64, dataType string, newBuckets []string, previousUpdatedAt int64) error { - query := ` - UPDATE file_data - SET delete_from_buckets = $1, updated_at = now_utc_micro_seconds() - WHERE file_id = $2 AND data_type = $3 AND updated_at = $4` - res, err := r.DB.ExecContext(ctx, query, pq.Array(newBuckets), fileID, dataType, previousUpdatedAt) - if err != nil { - return errors.Wrap(err, "failed to update delete from buckets") - } - - rowsAffected, err := res.RowsAffected() - if err != nil { - return errors.Wrap(err, "failed to check rows affected") - } - if rowsAffected == 0 { - return errors.New("no rows were updated, possible concurrent modification") - } - return nil -} - -// DeleteFileData deletes a file_data record by file_id and data_type if both replicated_buckets and delete_from_buckets are empty. -func (r *Repository) DeleteFileData(ctx context.Context, fileID int64, dataType string, previousUpdatedAt int64) error { - // First, check if both replicated_buckets and delete_from_buckets are empty. - var replicatedBuckets, deleteFromBuckets []string - query := `SELECT replicated_buckets, delete_from_buckets FROM file_data WHERE file_id = $1 AND data_type = $2` - err := r.DB.QueryRowContext(ctx, query, fileID, dataType).Scan(pq.Array(&replicatedBuckets), pq.Array(&deleteFromBuckets)) +func (r *Repository) GetFilesData(ctx context.Context, oType ente.ObjectType, fileIDs []int64) ([]filedata.Row, error) { + rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at + FROM file_data + WHERE data_type = $1 AND file_id = ANY($2)`, string(oType), pq.Array(fileIDs)) if err != nil { - if err == sql.ErrNoRows { - return errors.New("no file data found for the given file_id and data_type") - } - return errors.Wrap(err, "failed to check buckets before deleting file data") - } - - if len(replicatedBuckets) > 0 || len(deleteFromBuckets) > 0 { - return errors.New("cannot delete file data with non-empty replicated_buckets or delete_from_buckets") + return nil, stacktrace.Propagate(err, "") } + return convertRowsToFilesData(rows) - // Proceed with deletion if both arrays are empty and updated_at matches. - deleteQuery := `DELETE FROM file_data WHERE file_id = $1 AND data_type = $2 AND updated_at = $3` - res, err := r.DB.ExecContext(ctx, deleteQuery, fileID, dataType, previousUpdatedAt) - if err != nil { - return errors.Wrap(err, "failed to delete file data") - } - - rowsAffected, err := res.RowsAffected() - if err != nil { - return errors.Wrap(err, "failed to check rows affected") - } - if rowsAffected == 0 { - return errors.New("no rows were deleted, possible concurrent modification") - } - return nil } -// GetFileData retrieves a single file_data record by file_id and data_type. -func (r *Repository) GetFileData(ctx context.Context, fileID int64, dataType string) (FileData, error) { - var data FileData - query := `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at - FROM file_data - WHERE file_id = $1 AND data_type = $2` - err := r.DB.QueryRowContext(ctx, query, fileID, dataType).Scan( - &data.FileID, &data.UserID, &data.DataType, &data.Size, &data.LatestBucket, pq.Array(&data.ReplicatedBuckets), pq.Array(&data.DeleteFromBuckets), &data.PendingSync, &data.IsDeleted, &data.LastSyncTime, &data.CreatedAt, &data.UpdatedAt, - ) - if err != nil { - if err == sql.ErrNoRows { - return FileData{}, errors.Wrap(err, "no file data found") - } - return FileData{}, errors.Wrap(err, "failed to retrieve file data") - } - return data, nil -} - -// ListFileData retrieves all file_data records for a given user_id. -func (r *Repository) ListFileData(ctx context.Context, userID int64) ([]FileData, error) { - query := `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at - FROM file_data - WHERE user_id = $1` - rows, err := r.DB.QueryContext(ctx, query, userID) - if err != nil { - return nil, errors.Wrap(err, "failed to list file data") - } - defer rows.Close() - - var fileDataList []FileData +func convertRowsToFilesData(rows *sql.Rows) ([]filedata.Row, error) { + var filesData []filedata.Row for rows.Next() { - var data FileData - err := rows.Scan( - &data.FileID, &data.UserID, &data.DataType, &data.Size, &data.LatestBucket, pq.Array(&data.ReplicatedBuckets), pq.Array(&data.DeleteFromBuckets), &data.PendingSync, &data.IsDeleted, &data.LastSyncTime, &data.CreatedAt, &data.UpdatedAt, - ) + var fileData filedata.Row + err := rows.Scan(&fileData.FileID, &fileData.UserID, &fileData.Type, &fileData.Size, &fileData.LatestBucket, pq.Array(&fileData.ReplicatedBuckets), pq.Array(&fileData.DeleteFromBuckets), &fileData.PendingSync, &fileData.IsDeleted, &fileData.LastSyncTime, &fileData.CreatedAt, &fileData.UpdatedAt) if err != nil { - return nil, errors.Wrap(err, "failed to scan file data row") + return nil, stacktrace.Propagate(err, "") } - fileDataList = append(fileDataList, data) + filesData = append(filesData, fileData) } - if err = rows.Err(); err != nil { - return nil, errors.Wrap(err, "error iterating file data rows") - } - return fileDataList, nil + return filesData, nil + } From e32cd7b64ca2da911bfb7c56be9ad698c9dd317b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 2 Aug 2024 15:09:52 +0530 Subject: [PATCH 012/211] [server] Expose API to get and put metadata --- server/cmd/museum/main.go | 2 + server/pkg/api/file.go | 2 + server/pkg/api/file_data.go | 42 ++++++++++++++++++++ server/pkg/api/file_preview.go | 1 - server/pkg/controller/filedata/controller.go | 9 +++-- server/pkg/utils/s3config/s3config.go | 4 ++ 6 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 server/pkg/api/file_data.go delete mode 100644 server/pkg/api/file_preview.go diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index fda21bc659..bedcdf8eb0 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -413,6 +413,8 @@ func main() { privateAPI.POST("/files/file-data/playlist", fileHandler.ReportVideoPlayList) privateAPI.GET("/files/file-data/preview/upload-url/:fileID", fileHandler.GetVideoUploadURL) privateAPI.GET("/files/file-data/preview/:fileID", fileHandler.GetVideoPreviewUrl) + privateAPI.PUT("/files/data/", fileHandler.PutFileData) + privateAPI.POST("files/fetch-data/", fileHandler.GetFilesData) privateAPI.POST("/files", fileHandler.CreateOrUpdate) privateAPI.POST("/files/copy", fileHandler.CopyFiles) diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go index d2d12b91f7..2d00b8eb53 100644 --- a/server/pkg/api/file.go +++ b/server/pkg/api/file.go @@ -3,6 +3,7 @@ package api import ( "fmt" "github.com/ente-io/museum/pkg/controller/file_copy" + "github.com/ente-io/museum/pkg/controller/filedata" "net/http" "os" "strconv" @@ -24,6 +25,7 @@ import ( type FileHandler struct { Controller *controller.FileController FileCopyCtrl *file_copy.FileCopyController + FileDataCtrl *filedata.Controller } // DefaultMaxBatchSize is the default maximum API batch size unless specified otherwise diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go new file mode 100644 index 0000000000..81546b52ef --- /dev/null +++ b/server/pkg/api/file_data.go @@ -0,0 +1,42 @@ +package api + +import ( + "github.com/ente-io/museum/ente" + fileData "github.com/ente-io/museum/ente/filedata" + "github.com/ente-io/museum/pkg/utils/handler" + "github.com/gin-gonic/gin" + "net/http" +) + +func (f *FileHandler) PutFileData(ctx *gin.Context) { + var req fileData.PutFileDataRequest + if err := ctx.ShouldBindJSON(&req); err != nil { + ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error())) + return + } + if err := req.Validate(); err != nil { + ctx.JSON(http.StatusBadRequest, err) + return + } + err := f.FileDataCtrl.InsertOrUpdate(ctx, &req) + if err != nil { + handler.Error(ctx, err) + + return + } + ctx.JSON(http.StatusOK, gin.H{}) +} + +func (f *FileHandler) GetFilesData(ctx *gin.Context) { + var req fileData.GetFilesData + if err := ctx.ShouldBindJSON(&req); err != nil { + ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error())) + return + } + resp, err := f.FileDataCtrl.GetFilesData(ctx, req) + if err != nil { + handler.Error(ctx, err) + return + } + ctx.JSON(http.StatusOK, resp) +} diff --git a/server/pkg/api/file_preview.go b/server/pkg/api/file_preview.go deleted file mode 100644 index 778f64ec17..0000000000 --- a/server/pkg/api/file_preview.go +++ /dev/null @@ -1 +0,0 @@ -package api diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index d07f90029e..c428aaebae 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -185,7 +185,8 @@ func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row, dc string) defer func() { <-globalFileFetchSemaphore }() // Release back to global semaphore s3FileMetadata, err := c.fetchS3FileMetadata(context.Background(), row, dc) if err != nil { - log.Error("error fetching embedding object: "+row.S3FileMetadataObjectKey(), err) + log.WithField("bucket", row.LatestBucket). + Error("error fetching embedding object: "+row.S3FileMetadataObjectKey(), err) embeddingObjects[i] = bulkS3MetaFetchResult{ err: err, dbEntry: row, @@ -203,10 +204,10 @@ func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row, dc string) return embeddingObjects, nil } -func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, dc string) (*fileData.S3FileMetadata, error) { +func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, _ string) (*fileData.S3FileMetadata, error) { opt := _defaultFetchConfig objectKey := row.S3FileMetadataObjectKey() - ctxLogger := log.WithField("objectKey", objectKey).WithField("dc", dc) + ctxLogger := log.WithField("objectKey", objectKey).WithField("dc", row.LatestBucket) totalAttempts := opt.RetryCount + 1 timeout := opt.InitialTimeout for i := 0; i < totalAttempts; i++ { @@ -222,7 +223,7 @@ func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, cancel() return nil, stacktrace.Propagate(ctx.Err(), "") default: - obj, err := c.downloadObject(fetchCtx, objectKey, dc) + obj, err := c.downloadObject(fetchCtx, objectKey, row.LatestBucket) cancel() // Ensure cancel is called to release resources if err == nil { if i > 0 { diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index 204ee4e06c..1ad51f963b 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -152,6 +152,10 @@ func (config *S3Config) GetBucket(dc string) *string { return &bucket } +func (config *S3Config) IsBucketActive(dc string) bool { + return config.buckets[dc] != "" +} + func (config *S3Config) GetS3Config(dc string) *aws.Config { return config.s3Configs[dc] } From 986c00f4e8e805006aee5bd2fd9ccd79109b3445 Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Fri, 2 Aug 2024 16:36:05 +0530 Subject: [PATCH 013/211] fix(auth/support): remove support email and redirect to github discussions --- .../play/listings/en-US/full_description.txt | 2 +- .../android/en-US/full_description.txt | 2 +- auth/lib/core/constants.dart | 2 - auth/lib/l10n/arb/app_en.arb | 3 +- .../account/login_pwd_verification_page.dart | 1 - .../lib/ui/account/password_reentry_page.dart | 1 - auth/lib/ui/code_error_widget.dart | 9 +--- auth/lib/ui/common/report_bug.dart | 1 - .../ui/settings/data/import/aegis_import.dart | 2 +- .../data/import/bitwarden_import.dart | 2 +- .../data/import/encrypted_ente_import.dart | 2 +- .../settings/data/import/lastpass_import.dart | 2 +- .../data/import/plain_text_import.dart | 2 +- .../data/import/raivo_plain_text_import.dart | 2 +- .../settings/data/import/two_fas_import.dart | 2 +- .../ui/settings/support_section_widget.dart | 26 +++++----- auth/lib/ui/two_factor_recovery_page.dart | 10 ++-- auth/lib/utils/dialog_util.dart | 14 ++++-- auth/lib/utils/email_util.dart | 48 +++++++++++-------- 19 files changed, 67 insertions(+), 66 deletions(-) diff --git a/auth/android/app/src/main/play/listings/en-US/full_description.txt b/auth/android/app/src/main/play/listings/en-US/full_description.txt index 0bfee84fa5..571ef641a2 100644 --- a/auth/android/app/src/main/play/listings/en-US/full_description.txt +++ b/auth/android/app/src/main/play/listings/en-US/full_description.txt @@ -36,5 +36,5 @@ file, that adheres to the above format. SUPPORT -If you need help, please reach out to support@ente.io, and a human will get in touch with you. +If you need help, please visit @ https://github.com/ente-io/ente/discussions/new?category=q-a, and someone will get in touch with you. If you have feature requests, please create an issue @ https://github.com/ente-io/ente diff --git a/auth/fastlane/metadata/android/en-US/full_description.txt b/auth/fastlane/metadata/android/en-US/full_description.txt index f3664176dc..54c4a53525 100644 --- a/auth/fastlane/metadata/android/en-US/full_description.txt +++ b/auth/fastlane/metadata/android/en-US/full_description.txt @@ -36,5 +36,5 @@ file, that adheres to the above format. SUPPORT -If you need help, please reach out to support@ente.io, and a human will get in touch with you. +If you need help, please visit @ https://github.com/ente-io/ente/discussions/new?category=q-a, and someone will get in touch with you. If you have feature requests, please create an issue @ https://github.com/ente-io/ente diff --git a/auth/lib/core/constants.dart b/auth/lib/core/constants.dart index 5685220ac3..d7db33728f 100644 --- a/auth/lib/core/constants.dart +++ b/auth/lib/core/constants.dart @@ -32,8 +32,6 @@ const mnemonicKeyWordCount = 24; // https://stackoverflow.com/a/61162219 const dragSensitivity = 8; -const supportEmail = 'support@ente.io'; - // Default values for various feature flags class FFDefault { static const bool enableStripe = true; diff --git a/auth/lib/l10n/arb/app_en.arb b/auth/lib/l10n/arb/app_en.arb index 94698bf0b2..00d64cdec3 100644 --- a/auth/lib/l10n/arb/app_en.arb +++ b/auth/lib/l10n/arb/app_en.arb @@ -465,5 +465,6 @@ "appLockDescription": "Choose between your device's default lock screen and a custom lock screen with a PIN or password.", "pinLock": "Pin lock", "enterPin": "Enter PIN", - "setNewPin": "Set new PIN" + "setNewPin": "Set new PIN", + "importFailureDescNew": "Could not parse the selected file." } \ No newline at end of file diff --git a/auth/lib/ui/account/login_pwd_verification_page.dart b/auth/lib/ui/account/login_pwd_verification_page.dart index 2d2754ec16..d5ae7a3dbc 100644 --- a/auth/lib/ui/account/login_pwd_verification_page.dart +++ b/auth/lib/ui/account/login_pwd_verification_page.dart @@ -182,7 +182,6 @@ class _LoginPasswordVerificationPageState await sendLogs( context, context.l10n.contactSupport, - "auth@ente.io", postShare: () {}, ); } diff --git a/auth/lib/ui/account/password_reentry_page.dart b/auth/lib/ui/account/password_reentry_page.dart index 261f41db50..600d6ada98 100644 --- a/auth/lib/ui/account/password_reentry_page.dart +++ b/auth/lib/ui/account/password_reentry_page.dart @@ -140,7 +140,6 @@ class _PasswordReentryPageState extends State { await sendLogs( context, context.l10n.contactSupport, - "support@ente.io", postShare: () {}, ); } diff --git a/auth/lib/ui/code_error_widget.dart b/auth/lib/ui/code_error_widget.dart index 41250177c1..701bd576fe 100644 --- a/auth/lib/ui/code_error_widget.dart +++ b/auth/lib/ui/code_error_widget.dart @@ -5,7 +5,7 @@ import 'package:ente_auth/theme/ente_theme.dart'; import 'package:ente_auth/ui/common/gradient_button.dart'; import 'package:ente_auth/ui/linear_progress_widget.dart'; import 'package:ente_auth/ui/tools/debug/raw_codes_viewer.dart'; -import 'package:ente_auth/utils/dialog_util.dart'; +import 'package:ente_auth/utils/email_util.dart'; import 'package:flutter/material.dart'; class CodeErrorWidget extends StatelessWidget { @@ -114,12 +114,7 @@ class CodeErrorWidget extends StatelessWidget { text: context.l10n.contactSupport, fontSize: 10, onTap: () async { - await showErrorDialog( - context, - context.l10n.contactSupport, - context.l10n - .contactSupportViaEmailMessage("support@ente.io"), - ); + await openSupportPage(null, null); }, borderWidth: 0.6, borderRadius: 6, diff --git a/auth/lib/ui/common/report_bug.dart b/auth/lib/ui/common/report_bug.dart index 7b41cf0eb9..ad3daf53a6 100644 --- a/auth/lib/ui/common/report_bug.dart +++ b/auth/lib/ui/common/report_bug.dart @@ -22,7 +22,6 @@ PopupMenuButton reportBugPopupMenu(BuildContext context) { await sendLogs( context, "Contact support", - "support@ente.io", postShare: () {}, ); } diff --git a/auth/lib/ui/settings/data/import/aegis_import.dart b/auth/lib/ui/settings/data/import/aegis_import.dart index 471ce943ce..cb5adb8d07 100644 --- a/auth/lib/ui/settings/data/import/aegis_import.dart +++ b/auth/lib/ui/settings/data/import/aegis_import.dart @@ -77,7 +77,7 @@ Future _pickAegisJsonFile(BuildContext context) async { await showErrorDialog( context, context.l10n.sorry, - "${context.l10n.importFailureDesc}\n Error: ${e.toString()}", + "${context.l10n.importFailureDescNew}\n Error: ${e.toString()}", ); } } diff --git a/auth/lib/ui/settings/data/import/bitwarden_import.dart b/auth/lib/ui/settings/data/import/bitwarden_import.dart index eec93dbdb4..56106df39e 100644 --- a/auth/lib/ui/settings/data/import/bitwarden_import.dart +++ b/auth/lib/ui/settings/data/import/bitwarden_import.dart @@ -68,7 +68,7 @@ Future _pickBitwardenJsonFile(BuildContext context) async { await showErrorDialog( context, context.l10n.sorry, - "${context.l10n.importFailureDesc}\n Error: ${e.toString()}", + "${context.l10n.importFailureDescNew}\n Error: ${e.toString()}", ); } } diff --git a/auth/lib/ui/settings/data/import/encrypted_ente_import.dart b/auth/lib/ui/settings/data/import/encrypted_ente_import.dart index 3d7896f88e..4b055d2cf9 100644 --- a/auth/lib/ui/settings/data/import/encrypted_ente_import.dart +++ b/auth/lib/ui/settings/data/import/encrypted_ente_import.dart @@ -150,7 +150,7 @@ Future _pickEnteJsonFile(BuildContext context) async { await showErrorDialog( context, context.l10n.sorry, - context.l10n.importFailureDesc, + context.l10n.importFailureDescNew, ); } } diff --git a/auth/lib/ui/settings/data/import/lastpass_import.dart b/auth/lib/ui/settings/data/import/lastpass_import.dart index 550f2af7e0..db30286299 100644 --- a/auth/lib/ui/settings/data/import/lastpass_import.dart +++ b/auth/lib/ui/settings/data/import/lastpass_import.dart @@ -67,7 +67,7 @@ Future _pickLastpassJsonFile(BuildContext context) async { await showErrorDialog( context, context.l10n.sorry, - "${context.l10n.importFailureDesc}\n Error: ${e.toString()}", + "${context.l10n.importFailureDescNew}\n Error: ${e.toString()}", ); } } diff --git a/auth/lib/ui/settings/data/import/plain_text_import.dart b/auth/lib/ui/settings/data/import/plain_text_import.dart index 6867584b0f..486813ed90 100644 --- a/auth/lib/ui/settings/data/import/plain_text_import.dart +++ b/auth/lib/ui/settings/data/import/plain_text_import.dart @@ -144,7 +144,7 @@ Future _pickImportFile(BuildContext context) async { await showErrorDialog( context, context.l10n.sorry, - context.l10n.importFailureDesc, + context.l10n.importFailureDescNew, ); } } diff --git a/auth/lib/ui/settings/data/import/raivo_plain_text_import.dart b/auth/lib/ui/settings/data/import/raivo_plain_text_import.dart index 1b00086fe7..547145f815 100644 --- a/auth/lib/ui/settings/data/import/raivo_plain_text_import.dart +++ b/auth/lib/ui/settings/data/import/raivo_plain_text_import.dart @@ -67,7 +67,7 @@ Future _pickRaivoJsonFile(BuildContext context) async { await showErrorDialog( context, context.l10n.sorry, - "${context.l10n.importFailureDesc}\n Error: ${e.toString()}", + "${context.l10n.importFailureDescNew}\n Error: ${e.toString()}", ); } } diff --git a/auth/lib/ui/settings/data/import/two_fas_import.dart b/auth/lib/ui/settings/data/import/two_fas_import.dart index dcec016d49..fa4ec5aede 100644 --- a/auth/lib/ui/settings/data/import/two_fas_import.dart +++ b/auth/lib/ui/settings/data/import/two_fas_import.dart @@ -72,7 +72,7 @@ Future _pick2FasFile(BuildContext context) async { await showErrorDialog( context, context.l10n.sorry, - "${context.l10n.importFailureDesc}\n Error: ${e.toString()}", + "${context.l10n.importFailureDescNew}\n Error: ${e.toString()}", ); } } diff --git a/auth/lib/ui/settings/support_section_widget.dart b/auth/lib/ui/settings/support_section_widget.dart index 1343d23476..b445748650 100644 --- a/auth/lib/ui/settings/support_section_widget.dart +++ b/auth/lib/ui/settings/support_section_widget.dart @@ -70,18 +70,18 @@ class _SupportSectionWidgetState extends State { }, ), sectionOptionSpacing, - MenuItemWidget( - captionedTextWidget: CaptionedTextWidget( - title: l10n.email, - ), - pressedColor: getEnteColorScheme(context).fillFaint, - trailingIcon: Icons.chevron_right_outlined, - trailingIconIsMuted: true, - onTap: () async { - await sendEmail(context, to: supportEmail); - }, - ), - sectionOptionSpacing, + // MenuItemWidget( + // captionedTextWidget: CaptionedTextWidget( + // title: l10n.email, + // ), + // pressedColor: getEnteColorScheme(context).fillFaint, + // trailingIcon: Icons.chevron_right_outlined, + // trailingIconIsMuted: true, + // onTap: () async { + // await sendEmail(context, to: supportEmail); + // }, + // ), + // sectionOptionSpacing, MenuItemWidget( captionedTextWidget: CaptionedTextWidget( title: l10n.reportABug, @@ -90,7 +90,7 @@ class _SupportSectionWidgetState extends State { trailingIcon: Icons.chevron_right_outlined, trailingIconIsMuted: true, onTap: () async { - await sendLogs(context, l10n.reportBug, "auth@ente.io"); + await sendLogs(context, l10n.reportBug); }, onDoubleTap: () async { try { diff --git a/auth/lib/ui/two_factor_recovery_page.dart b/auth/lib/ui/two_factor_recovery_page.dart index 5743f0246f..c458c001e5 100644 --- a/auth/lib/ui/two_factor_recovery_page.dart +++ b/auth/lib/ui/two_factor_recovery_page.dart @@ -1,7 +1,7 @@ import 'package:ente_auth/l10n/l10n.dart'; import 'package:ente_auth/models/account/two_factor.dart'; import 'package:ente_auth/services/user_service.dart'; -import 'package:ente_auth/utils/dialog_util.dart'; +import 'package:ente_auth/utils/email_util.dart'; import 'package:flutter/material.dart'; class TwoFactorRecoveryPage extends StatefulWidget { @@ -86,12 +86,8 @@ class _TwoFactorRecoveryPageState extends State { ), GestureDetector( behavior: HitTestBehavior.translucent, - onTap: () { - showErrorDialog( - context, - l10n.contactSupport, - l10n.contactSupportViaEmailMessage("support@ente.io"), - ); + onTap: () async { + await openSupportPage(null, null); }, child: Container( padding: const EdgeInsets.all(40), diff --git a/auth/lib/utils/dialog_util.dart b/auth/lib/utils/dialog_util.dart index 24636bf889..158db3dec1 100644 --- a/auth/lib/utils/dialog_util.dart +++ b/auth/lib/utils/dialog_util.dart @@ -31,12 +31,21 @@ Future showErrorDialog( title: title, body: body, isDismissible: isDismissable, - buttons: const [ + buttons: [ ButtonWidget( + buttonType: ButtonType.primary, + labelText: context.l10n.contactSupport, + isInAlert: true, + buttonAction: ButtonAction.first, + onTap: () async { + await openSupportPage(body, null); + }, + ), + const ButtonWidget( buttonType: ButtonType.secondary, labelText: "OK", isInAlert: true, - buttonAction: ButtonAction.first, + buttonAction: ButtonAction.second, ), ], ); @@ -158,7 +167,6 @@ Future showGenericErrorDialog({ await sendLogs( context, context.l10n.contactSupport, - "support@ente.io", postShare: () {}, ); }, diff --git a/auth/lib/utils/email_util.dart b/auth/lib/utils/email_util.dart index 8b04122289..73291a1ed5 100644 --- a/auth/lib/utils/email_util.dart +++ b/auth/lib/utils/email_util.dart @@ -16,7 +16,6 @@ import 'package:ente_auth/utils/toast_util.dart'; import "package:file_saver/file_saver.dart"; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; -import 'package:flutter_email_sender/flutter_email_sender.dart'; import "package:intl/intl.dart"; import 'package:logging/logging.dart'; import 'package:package_info_plus/package_info_plus.dart'; @@ -35,8 +34,7 @@ bool isValidEmail(String? email) { Future sendLogs( BuildContext context, - String title, - String toEmail, { + String title, { Function? postShare, String? subject, String? body, @@ -55,7 +53,7 @@ Future sendLogs( buttonAction: ButtonAction.first, shouldSurfaceExecutionStates: false, onTap: () async { - await _sendLogs(context, toEmail, subject, body); + await openSupportPage(subject, body); if (postShare != null) { postShare(); } @@ -111,27 +109,35 @@ Future sendLogs( ); } -Future _sendLogs( - BuildContext context, - String toEmail, +Future openSupportPage( String? subject, String? body, ) async { - final String zipFilePath = await getZippedLogsFile(context); - final Email email = Email( - recipients: [toEmail], - subject: subject ?? '', - body: body ?? '', - attachmentPaths: [zipFilePath], - isHTML: false, - ); - try { - await FlutterEmailSender.send(email); - } catch (e, s) { - _logger.severe('email sender failed', e, s); - Navigator.of(context, rootNavigator: true).pop(); - await shareLogs(context, toEmail, zipFilePath); + const url = "https://github.com/ente-io/ente/discussions/new?category=q-a"; + if (subject != null && body != null) { + await launchUrl( + Uri.parse( + "$url&title=$subject&body=$body", + ), + ); + } else { + await launchUrl(Uri.parse(url)); } + // final String zipFilePath = await getZippedLogsFile(context); + // final Email email = Email( + // recipients: [toEmail], + // subject: subject ?? '', + // body: body ?? '', + // attachmentPaths: [zipFilePath], + // isHTML: false, + // ); + // try { + // await FlutterEmailSender.send(email); + // } catch (e, s) { + // _logger.severe('email sender failed', e, s); + // Navigator.of(context, rootNavigator: true).pop(); + // await shareLogs(context, toEmail, zipFilePath); + // } } Future getZippedLogsFile(BuildContext context) async { From 289718f7f6c82eb9d30ac84e19b4ef2d6b147d8a Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Fri, 2 Aug 2024 16:40:39 +0530 Subject: [PATCH 014/211] fix(auth/support): update support email to auth@ente.io --- auth/linux/packaging/deb/make_config.yaml | 2 +- auth/linux/packaging/ente_auth.appdata.xml | 2 +- auth/linux/packaging/pacman/make_config.yaml | 2 +- auth/linux/packaging/rpm/make_config.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/auth/linux/packaging/deb/make_config.yaml b/auth/linux/packaging/deb/make_config.yaml index 7092994bd4..3ec44bae56 100644 --- a/auth/linux/packaging/deb/make_config.yaml +++ b/auth/linux/packaging/deb/make_config.yaml @@ -2,7 +2,7 @@ display_name: Auth package_name: auth maintainer: name: Ente.io Developers - email: human@ente.io + email: auth@ente.io priority: optional section: x11 essential: false diff --git a/auth/linux/packaging/ente_auth.appdata.xml b/auth/linux/packaging/ente_auth.appdata.xml index c218b5ce48..5ef4c647f4 100644 --- a/auth/linux/packaging/ente_auth.appdata.xml +++ b/auth/linux/packaging/ente_auth.appdata.xml @@ -27,5 +27,5 @@ Ente.io Developers - human@ente.io + auth@ente.io \ No newline at end of file diff --git a/auth/linux/packaging/pacman/make_config.yaml b/auth/linux/packaging/pacman/make_config.yaml index c27c1c703f..7545d8c06e 100644 --- a/auth/linux/packaging/pacman/make_config.yaml +++ b/auth/linux/packaging/pacman/make_config.yaml @@ -2,7 +2,7 @@ display_name: Auth package_name: auth maintainer: name: Ente.io Developers - email: human@ente.io + email: auth@ente.io licenses: - GPLv3 icon: assets/icons/auth-icon.png diff --git a/auth/linux/packaging/rpm/make_config.yaml b/auth/linux/packaging/rpm/make_config.yaml index c285b90b30..6f6c45153a 100644 --- a/auth/linux/packaging/rpm/make_config.yaml +++ b/auth/linux/packaging/rpm/make_config.yaml @@ -3,7 +3,7 @@ summary: 2FA app with free end-to-end encrypted backup and sync group: Application/Utility vendor: Ente.io packager: Ente.io Developers -packagerEmail: human@ente.io +packagerEmail: auth@ente.io license: GPLv3 url: https://github.com/ente-io/ente From d1dc977d5e24c16bd89541f594749c35a0c760dd Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 3 Aug 2024 16:06:47 +0530 Subject: [PATCH 015/211] [server] Add struct for fileData config --- server/configurations/local.yaml | 9 +++++++++ server/pkg/utils/s3config/filedata.go | 12 ++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 server/pkg/utils/s3config/filedata.go diff --git a/server/configurations/local.yaml b/server/configurations/local.yaml index f392663c0e..52db52ca07 100644 --- a/server/configurations/local.yaml +++ b/server/configurations/local.yaml @@ -169,6 +169,15 @@ s3: # resolved, e.g. when running a local instance, or when using MinIO as a # production store. #use_path_style_urls: true + # + # file-data-storage: + # derivedMetadata: + # bucket: + # replicas: [] + # img_preview: + # bucket: + # replicas: [] + # Key used for encrypting customer emails before storing them in DB # diff --git a/server/pkg/utils/s3config/filedata.go b/server/pkg/utils/s3config/filedata.go new file mode 100644 index 0000000000..8874d18ee2 --- /dev/null +++ b/server/pkg/utils/s3config/filedata.go @@ -0,0 +1,12 @@ +package s3config + +import "github.com/ente-io/museum/ente" + +type ObjectBucketConfig struct { + PrimaryBucket string `mapstructure:"primary"` + ReplicaBuckets []string `mapstructure:"replicas"` +} + +type FileDataConfig struct { + ObjectBucketConfig map[ente.ObjectType]ObjectBucketConfig `mapstructure:"objectBuckets"` +} From 18d58a9eeebc1f9d8925f5966eed80f64f37eaf9 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 5 Aug 2024 16:44:01 +0530 Subject: [PATCH 016/211] [server] Parse config for file-data buckets --- server/configurations/local.yaml | 8 ++--- server/pkg/utils/s3config/filedata.go | 35 +++++++++++++++++--- server/pkg/utils/s3config/s3config.go | 47 ++++++++++++++++++++------- 3 files changed, 71 insertions(+), 19 deletions(-) diff --git a/server/configurations/local.yaml b/server/configurations/local.yaml index 9d724aaf04..9e26d820f8 100644 --- a/server/configurations/local.yaml +++ b/server/configurations/local.yaml @@ -175,11 +175,11 @@ s3: # # file-data-storage: # derivedMetadata: - # bucket: - # replicas: [] + # primaryBucket: + # replicaBuckets: [] # img_preview: - # bucket: - # replicas: [] + # primaryBucket: + # replicaBuckets: [] # Key used for encrypting customer emails before storing them in DB diff --git a/server/pkg/utils/s3config/filedata.go b/server/pkg/utils/s3config/filedata.go index 8874d18ee2..a39501ba40 100644 --- a/server/pkg/utils/s3config/filedata.go +++ b/server/pkg/utils/s3config/filedata.go @@ -1,12 +1,39 @@ package s3config -import "github.com/ente-io/museum/ente" +import ( + "fmt" + "github.com/ente-io/museum/ente" +) type ObjectBucketConfig struct { - PrimaryBucket string `mapstructure:"primary"` - ReplicaBuckets []string `mapstructure:"replicas"` + PrimaryBucket string `mapstructure:"primaryBucket"` + ReplicaBuckets []string `mapstructure:"replicaBuckets"` } type FileDataConfig struct { - ObjectBucketConfig map[ente.ObjectType]ObjectBucketConfig `mapstructure:"objectBuckets"` + ObjectBucketConfig map[ente.ObjectType]ObjectBucketConfig `mapstructure:"file-data-config"` +} + +func (f FileDataConfig) HasConfig(objectType ente.ObjectType) bool { + if objectType == "" || objectType == ente.FILE || objectType == ente.THUMBNAIL { + panic(fmt.Sprintf("Invalid object type: %s", objectType)) + } + _, ok := f.ObjectBucketConfig[objectType] + return ok +} + +func (f FileDataConfig) GetPrimaryBucketID(objectType ente.ObjectType) string { + config, ok := f.ObjectBucketConfig[objectType] + if !ok { + panic(fmt.Sprintf("No config for object type: %s, use HasConfig", objectType)) + } + return config.PrimaryBucket +} + +func (f FileDataConfig) GetReplicaBuckets(objectType ente.ObjectType) []string { + config, ok := f.ObjectBucketConfig[objectType] + if !ok { + panic(fmt.Sprintf("No config for object type: %s, use HasConfig", objectType)) + } + return config.ReplicaBuckets } diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index 1ad51f963b..a6af453574 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -1,10 +1,12 @@ package s3config import ( + "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" + "github.com/ente-io/museum/ente" log "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -39,10 +41,18 @@ type S3Config struct { // Indicates if local minio buckets are being used. Enables various // debugging workarounds; not tested/intended for production. areLocalBuckets bool + + // FileDataConfig is the configuration for various file data. + // If for particular object type, the bucket is not specified, it will + // default to hotDC as the bucket with no replicas. Initially, this config won't support + // existing objectType (file, thumbnail) and will be used for new objectTypes. In the future, + // we can migrate existing objectTypes to this config. + fileDataConfig FileDataConfig } // # Datacenters -// +// Note: We are now renaming datacenter names to bucketID. Till the migration is completed, you will see usage of both +// terminology. // Below are some high level details about the three replicas ("data centers") // that are in use. There are a few other legacy ones too. // @@ -78,7 +88,6 @@ var ( ) // Number of days that the wasabi bucket is configured to retain objects. -// // We must wait at least these many days after removing the conditional hold // before we can delete the object. const WasabiObjectConditionalHoldDays = 21 @@ -119,7 +128,6 @@ func (config *S3Config) initialize() { config.areLocalBuckets = areLocalBuckets for _, dc := range dcs { - config.buckets[dc] = viper.GetString("s3." + dc + ".bucket") config.buckets[dc] = viper.GetString("s3." + dc + ".bucket") s3Config := aws.Config{ Credentials: credentials.NewStaticCredentials(viper.GetString("s3."+dc+".key"), @@ -145,23 +153,40 @@ func (config *S3Config) initialize() { config.isWasabiComplianceEnabled = viper.GetBool("s3." + dc + ".compliance") } } + + if err := viper.Sub("s3").Unmarshal(&config.fileDataConfig); err != nil { + log.Fatal("Unable to decode into struct: %v\n", err) + return + } + } -func (config *S3Config) GetBucket(dc string) *string { - bucket := config.buckets[dc] +func (config *S3Config) GetBucket(dcOrBucketID string) *string { + bucket := config.buckets[dcOrBucketID] return &bucket } -func (config *S3Config) IsBucketActive(dc string) bool { - return config.buckets[dc] != "" +// GetBucketID returns the bucket ID for the given object type. Note: existing dc are renamed as bucketID +func (config *S3Config) GetBucketID(oType ente.ObjectType) string { + if config.fileDataConfig.HasConfig(oType) { + return config.fileDataConfig.GetPrimaryBucketID(oType) + } + if oType == ente.DerivedMeta || oType == ente.PreviewVideo || oType == ente.PreviewImage { + return config.derivedStorageDC + } + panic(fmt.Sprintf("No bucket for object type: %s", oType)) +} + +func (config *S3Config) IsBucketActive(bucketID string) bool { + return config.buckets[bucketID] != "" } -func (config *S3Config) GetS3Config(dc string) *aws.Config { - return config.s3Configs[dc] +func (config *S3Config) GetS3Config(dcOrBucketID string) *aws.Config { + return config.s3Configs[dcOrBucketID] } -func (config *S3Config) GetS3Client(dc string) s3.S3 { - return config.s3Clients[dc] +func (config *S3Config) GetS3Client(dcOrBucketID string) s3.S3 { + return config.s3Clients[dcOrBucketID] } func (config *S3Config) GetHotDataCenter() string { From 27eb5ecc2bc2b38d99d5e666037defff1827c1f5 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 5 Aug 2024 16:44:34 +0530 Subject: [PATCH 017/211] [server] Update DB Script --- server/migrations/89_derived_data_table.down.sql | 5 ++++- server/migrations/89_derived_data_table.up.sql | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/server/migrations/89_derived_data_table.down.sql b/server/migrations/89_derived_data_table.down.sql index 1d39d0663f..024a1da670 100644 --- a/server/migrations/89_derived_data_table.down.sql +++ b/server/migrations/89_derived_data_table.down.sql @@ -3,4 +3,7 @@ DROP INDEX IF EXISTS idx_file_data_user_type_deleted; DROP TABLE IF EXISTS file_data; -DROP TYPE IF EXISTS file_data_type; \ No newline at end of file +DROP TYPE IF EXISTS file_data_type; + +-- Delete triggers +DROP TRIGGER IF EXISTS check_no_common_entries ON file_data; \ No newline at end of file diff --git a/server/migrations/89_derived_data_table.up.sql b/server/migrations/89_derived_data_table.up.sql index 042f46c564..53b0ae95f3 100644 --- a/server/migrations/89_derived_data_table.up.sql +++ b/server/migrations/89_derived_data_table.up.sql @@ -1,5 +1,5 @@ -- Create the derived table -CREATE TABLE file_data +CREATE TABLE IF NOT EXISTS file_data ( file_id BIGINT NOT NULL, user_id BIGINT NOT NULL, From 81c3626c6c99794cad43a0841eb90e2f6ca5734b Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 5 Aug 2024 17:31:12 +0530 Subject: [PATCH 018/211] Rename --- server/pkg/utils/s3config/s3config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index a6af453574..b6ec9ba2af 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -142,11 +142,11 @@ func (config *S3Config) initialize() { s3Config.DisableSSL = aws.Bool(true) s3Config.S3ForcePathStyle = aws.Bool(true) } - session, err := session.NewSession(&s3Config) + s3Session, err := session.NewSession(&s3Config) if err != nil { log.Fatal("Could not create session for " + dc) } - s3Client := *s3.New(session) + s3Client := *s3.New(s3Session) config.s3Configs[dc] = &s3Config config.s3Clients[dc] = s3Client if dc == dcWasabiEuropeCentral_v3 { From 744d6bc6eacfab7d5831d0c589f6c849f24b28c9 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 5 Aug 2024 17:35:05 +0530 Subject: [PATCH 019/211] Add helper method to empty bucket for given user --- server/cmd/museum/main.go | 1 + server/pkg/utils/s3config/s3config.go | 58 +++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index 17740becc0..c56045e4a4 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -134,6 +134,7 @@ func main() { }, []string{"method"}) s3Config := s3config.NewS3Config() + //s3Config.EmptyB5Bucket(1580559962386453) passkeysRepo, err := passkey.NewRepository(db) if err != nil { diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index b6ec9ba2af..b0f88ee958 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -217,6 +217,64 @@ func (config *S3Config) GetDerivedStorageBucket() *string { return config.GetBucket(config.derivedStorageDC) } +func (config *S3Config) EmptyB5Bucket(userID int64) { + bucket := config.GetBucket("b5") + prefix := fmt.Sprintf("%d/", userID) + // Create an S3 client + s3Client := config.GetS3Client("b5") + + // List objects with the specified prefix + listObjectsInput := &s3.ListObjectsV2Input{ + Bucket: aws.String(*bucket), + Prefix: aws.String(prefix), + } + + for { + // Get the list of objects + listObjectsOutput, err := s3Client.ListObjectsV2(listObjectsInput) + if err != nil { + log.Fatalf("unable to list objects, %v", err) + } + + if len(listObjectsOutput.Contents) == 0 { + fmt.Println("No objects found with the specified prefix.") + break + } + + // Create a slice of object identifiers to delete + var objectIdentifiers []*s3.ObjectIdentifier + for _, object := range listObjectsOutput.Contents { + objectIdentifiers = append(objectIdentifiers, &s3.ObjectIdentifier{ + Key: object.Key, + }) + } + + // Delete the objects + deleteObjectsInput := &s3.DeleteObjectsInput{ + Bucket: aws.String(*bucket), + Delete: &s3.Delete{ + Objects: objectIdentifiers, + Quiet: aws.Bool(true), + }, + } + + _, err = s3Client.DeleteObjects(deleteObjectsInput) + if err != nil { + log.Fatalf("unable to delete objects, %v", err) + } + + fmt.Printf("Deleted %d objects.\n", len(objectIdentifiers)) + + // If there are more objects to list, set the continuation token for the next request + if *listObjectsOutput.IsTruncated { + listObjectsInput.ContinuationToken = listObjectsOutput.NextContinuationToken + } else { + break + } + } + fmt.Println("All objects with the specified prefix have been deleted.") +} + func (config *S3Config) GetDerivedStorageS3Client() *s3.S3 { s3Client := config.GetS3Client(config.derivedStorageDC) return &s3Client From 543aa6b9cf4cc841e1915e024ce6f00da4b72168 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 5 Aug 2024 17:46:00 +0530 Subject: [PATCH 020/211] Clean up --- server/pkg/controller/filedata/controller.go | 56 ++++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index c428aaebae..fb795c01c0 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -44,18 +44,17 @@ type bulkS3MetaFetchResult struct { } type Controller struct { - Repo *fileDataRepo.Repository - AccessCtrl access.Controller - ObjectCleanupController *controller.ObjectCleanupController - S3Config *s3config.S3Config - QueueRepo *repo.QueueRepository - TaskLockingRepo *repo.TaskLockRepository - FileRepo *repo.FileRepository - CollectionRepo *repo.CollectionRepository - HostName string - cleanupCronRunning bool - derivedStorageDataCenter string - downloadManagerCache map[string]*s3manager.Downloader + Repo *fileDataRepo.Repository + AccessCtrl access.Controller + ObjectCleanupController *controller.ObjectCleanupController + S3Config *s3config.S3Config + QueueRepo *repo.QueueRepository + TaskLockingRepo *repo.TaskLockRepository + FileRepo *repo.FileRepository + CollectionRepo *repo.CollectionRepository + HostName string + cleanupCronRunning bool + downloadManagerCache map[string]*s3manager.Downloader } func New(repo *fileDataRepo.Repository, @@ -67,24 +66,23 @@ func New(repo *fileDataRepo.Repository, fileRepo *repo.FileRepository, collectionRepo *repo.CollectionRepository, hostName string) *Controller { - embeddingDcs := []string{s3Config.GetHotBackblazeDC(), s3Config.GetHotWasabiDC(), s3Config.GetWasabiDerivedDC(), s3Config.GetDerivedStorageDataCenter()} + embeddingDcs := []string{s3Config.GetHotBackblazeDC(), s3Config.GetHotWasabiDC(), s3Config.GetWasabiDerivedDC(), s3Config.GetDerivedStorageDataCenter(), "b5"} cache := make(map[string]*s3manager.Downloader, len(embeddingDcs)) for i := range embeddingDcs { s3Client := s3Config.GetS3Client(embeddingDcs[i]) cache[embeddingDcs[i]] = s3manager.NewDownloaderWithClient(&s3Client) } return &Controller{ - Repo: repo, - AccessCtrl: accessCtrl, - ObjectCleanupController: objectCleanupController, - S3Config: s3Config, - QueueRepo: queueRepo, - TaskLockingRepo: taskLockingRepo, - FileRepo: fileRepo, - CollectionRepo: collectionRepo, - HostName: hostName, - derivedStorageDataCenter: s3Config.GetDerivedStorageDataCenter(), - downloadManagerCache: cache, + Repo: repo, + AccessCtrl: accessCtrl, + ObjectCleanupController: objectCleanupController, + S3Config: s3Config, + QueueRepo: queueRepo, + TaskLockingRepo: taskLockingRepo, + FileRepo: fileRepo, + CollectionRepo: collectionRepo, + HostName: hostName, + downloadManagerCache: cache, } } @@ -105,7 +103,8 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR DecryptionHeader: *req.DecryptionHeader, Client: network.GetClientInfo(ctx), } - size, uploadErr := c.uploadObject(obj, objectKey, c.derivedStorageDataCenter) + bucketID := c.S3Config.GetBucketID(req.Type) + size, uploadErr := c.uploadObject(obj, objectKey, bucketID) if uploadErr != nil { log.Error(uploadErr) return stacktrace.Propagate(uploadErr, "") @@ -115,7 +114,7 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR Type: req.Type, UserID: fileOwnerID, Size: size, - LatestBucket: c.derivedStorageDataCenter, + LatestBucket: bucketID, } err = c.Repo.InsertOrUpdate(ctx, row) if err != nil { @@ -146,7 +145,7 @@ func (c *Controller) GetFilesData(ctx *gin.Context, req fileData.GetFilesData) ( } pendingIndexFileIds := array.FindMissingElementsInSecondList(req.FileIDs, dbFileIds) // Fetch missing doRows in parallel - s3MetaFetchResults, err := c.getS3FileMetadataParallel(activeRows, c.derivedStorageDataCenter) + s3MetaFetchResults, err := c.getS3FileMetadataParallel(activeRows) if err != nil { return nil, stacktrace.Propagate(err, "") } @@ -173,7 +172,7 @@ func (c *Controller) GetFilesData(ctx *gin.Context, req fileData.GetFilesData) ( }, nil } -func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row, dc string) ([]bulkS3MetaFetchResult, error) { +func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row) ([]bulkS3MetaFetchResult, error) { var wg sync.WaitGroup embeddingObjects := make([]bulkS3MetaFetchResult, len(dbRows)) for i, _ := range dbRows { @@ -183,6 +182,7 @@ func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row, dc string) go func(i int, row fileData.Row) { defer wg.Done() defer func() { <-globalFileFetchSemaphore }() // Release back to global semaphore + dc := row.LatestBucket s3FileMetadata, err := c.fetchS3FileMetadata(context.Background(), row, dc) if err != nil { log.WithField("bucket", row.LatestBucket). From 1570b0a551403ec56e8cd224c03be52354e914dc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 13:06:26 +0530 Subject: [PATCH 021/211] Entry point --- web/packages/media/file-metadata.ts | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index cecbe8765f..6dcde2c01f 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -164,6 +164,31 @@ export interface PublicMagicMetadata { h?: number; } +/** + * Update the public magic metadata associated with a file on remote. + * + * This function updates the public magic metadata on remote, and also returns a + * new {@link EnteFile} object with the updated values, but it does not update + * the state of the local databases. The caller needs to ensure that we + * subsequently sync with remote to fetch the updates as part of the diff and + * update the {@link EnteFile} that is persisted in our local database. + * + * @param enteFile The {@link EnteFile} whose public magic metadata we want to + * update. + * + * @param updatedMetadata A subset of {@link PublicMagicMetadata} containing the + * fields that we want to add or update. + * + * @param encryptMetadataF A function that is used to encrypt the metadata. + * + * @returns A {@link EnteFile} object with the updated public magic metadata. + */ +export const updateRemotePublicMagicMetadata = async ( + enteFile: EnteFile, + updatedMetadata: Partial, + encryptMetadataF: EncryptMetadataF, +) => {}; + /** * Magic metadata, either public and private, as persisted and used by remote. * From 1c4ae46270438d2b95c0fffa9c524a8e8f2ef990 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 13:53:55 +0530 Subject: [PATCH 022/211] wip 1 --- web/packages/media/file-metadata.ts | 98 ++++++++++++++++--- .../shared/crypto/internal/crypto.worker.ts | 10 +- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 6dcde2c01f..d39412b8b7 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -1,8 +1,10 @@ -import { encryptMetadata } from "@/base/crypto/ente"; +import { encryptMetadata, type decryptMetadata } from "@/base/crypto/ente"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; import { type EnteFile } from "@/new/photos/types/file"; import { FileType } from "./file-type"; +import { z} from 'zod'; +import { nullToUndefined } from "@/utils/transform"; /** * Information about the file that never changes post upload. @@ -164,6 +166,79 @@ export interface PublicMagicMetadata { h?: number; } +/** + * Zod schema for the {@link PublicMagicMetadata} type. + * + * See: [Note: Duplicated Zod schema and TypeScript type] + * + * --- + * + * [Note: Use passthrough for metadata Zod schemas] + * + * It is important to (recursively) use the {@link passthrough} option when + * definining Zod schemas for the various metadata types (the plaintext JSON + * objects) because we want to retain all the fields we get from remote. There + * might be other, newer, clients out there adding fields that the current + * client might not we aware of, and we don't want to overwrite them. + */ +const PublicMagicMetadata = z.object({ + editedTime: z.number().nullish().transform(nullToUndefined), +}).passthrough(); + +/** + * A function that can be used to encrypt the contents of a metadata field + * associated with a file. + * + * This is parameterized to allow us to use either the regular + * {@link encryptMetadata} (if we're already running in a web worker) or its web + * worker wrapper (if we're running on the main thread). + */ +export type EncryptMetadataF = typeof encryptMetadata; + +/** + * A function that can be used to decrypt the contents of a metadata field + * associated with a file. + * + * This is parameterized to allow us to use either the regular + * {@link encryptMetadata} (if we're already running in a web worker) or its web + * worker wrapper (if we're running on the main thread). + */ +export type DecryptMetadataF = typeof decryptMetadata; + +/** + * Return the public magic metadata for the given {@link enteFile}. + * + * The file we persist in our local db has the metadata in the encrypted form + * that we get it from remote. We decrypt when we read it, and also hang the + * decrypted version to the in-memory {@link EnteFile} as a cache. + * + * If the file doesn't have any public magic metadata attached to it, return + * `undefined`. + */ +export const decryptPublicMagicMetadata = async (enteFile: EnteFile, decryptMetadataF: DecryptMetadataF): Promise => { + const envelope = enteFile.pubMagicMetadata; + if (!envelope) return undefined; + + // TODO: This function can be optimized to directly return the cached value + // instead of reparsing it using Zod. But that requires us (a) first fix the + // types, and (b) guarantee that we're the only ones putting that parsed + // data there, so that it is in a known good state (currently we exist in + // parallel with other functions that do the similar things). + + const jsonValue = typeof envelope.data == "string" ? decryptMetadataF(envelope.data, envelope.header, enteFile.key) : envelope.data; + const result = PublicMagicMetadata.parse(withoutNullAndUndefinedValues(jsonValue)); + + envelope.data = result; + + return result; +} + +const withoutNullAndUndefinedValues = (o: {}) => + Object.fromEntries(Object.entries(o).filter( + ([, v]) => v !== null && v !== undefined, + )); + + /** * Update the public magic metadata associated with a file on remote. * @@ -171,12 +246,12 @@ export interface PublicMagicMetadata { * new {@link EnteFile} object with the updated values, but it does not update * the state of the local databases. The caller needs to ensure that we * subsequently sync with remote to fetch the updates as part of the diff and - * update the {@link EnteFile} that is persisted in our local database. + * update the {@link EnteFile} that is persisted in our local db. * * @param enteFile The {@link EnteFile} whose public magic metadata we want to * update. * - * @param updatedMetadata A subset of {@link PublicMagicMetadata} containing the + * @param metadataUpdates A subset of {@link PublicMagicMetadata} containing the * fields that we want to add or update. * * @param encryptMetadataF A function that is used to encrypt the metadata. @@ -185,9 +260,13 @@ export interface PublicMagicMetadata { */ export const updateRemotePublicMagicMetadata = async ( enteFile: EnteFile, - updatedMetadata: Partial, + metadataUpdates: Partial, encryptMetadataF: EncryptMetadataF, -) => {}; +) => { + const updatedMetadata = { + ...file. + } +}; /** * Magic metadata, either public and private, as persisted and used by remote. @@ -246,15 +325,6 @@ interface UpdateMagicMetadataRequest { }[]; } -/** - * A function that can be used to encrypt the contents of a metadata field - * associated with a file. - * - * This is parameterized to allow us to use either the regular - * {@link encryptMetadata} or the web worker wrapper for it. - */ -export type EncryptMetadataF = typeof encryptMetadata; - /** * Construct an remote update request payload from the public or private magic * metadata JSON object for an {@link enteFile}, using the provided diff --git a/web/packages/shared/crypto/internal/crypto.worker.ts b/web/packages/shared/crypto/internal/crypto.worker.ts index d825ba5a57..356bde8580 100644 --- a/web/packages/shared/crypto/internal/crypto.worker.ts +++ b/web/packages/shared/crypto/internal/crypto.worker.ts @@ -8,12 +8,16 @@ import type { StateAddress } from "libsodium-wrappers"; * specific layer (base/crypto/ente.ts) or the internal libsodium layer * (internal/libsodium.ts). * - * Running these in a web worker allows us to use potentially CPU-intensive - * crypto operations from the main thread without stalling the UI. + * Use these when running on the main thread, since running these in a web + * worker allows us to use potentially CPU-intensive crypto operations from the + * main thread without stalling the UI. + * + * If the code that needs this functionality is already running in the context + * of a web worker, then use the underlying functions directly. * * See: [Note: Crypto code hierarchy]. * - * Note: Keep these methods logic free. They should just act as trivial proxies. + * Note: Keep these methods logic free. They are meant to be trivial proxies. */ export class DedicatedCryptoWorker { async decryptThumbnail( From a61ea9338efd16a5d0e16a95cf6246ddfa0d5a90 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 14:06:51 +0530 Subject: [PATCH 023/211] Add note about Zod and exactOptionalPropertyTypes --- web/packages/media/file-metadata.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index d39412b8b7..a2205e991c 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -182,7 +182,15 @@ export interface PublicMagicMetadata { * client might not we aware of, and we don't want to overwrite them. */ const PublicMagicMetadata = z.object({ - editedTime: z.number().nullish().transform(nullToUndefined), + // [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] + // + // Using `optional` is accurate here. The key is optional, but the value + // itself is not optional. Zod doesn't work with + // `exactOptionalPropertyTypes` yet, but it seems to be on the roadmap so we + // suppress these mismatches. + // + // See: https://github.com/colinhacks/zod/issues/635#issuecomment-2196579063 + editedTime: z.number().optional(), }).passthrough(); /** @@ -228,8 +236,10 @@ export const decryptPublicMagicMetadata = async (enteFile: EnteFile, decryptMeta const jsonValue = typeof envelope.data == "string" ? decryptMetadataF(envelope.data, envelope.header, enteFile.key) : envelope.data; const result = PublicMagicMetadata.parse(withoutNullAndUndefinedValues(jsonValue)); + // @ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] envelope.data = result; + // @ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] return result; } @@ -238,7 +248,6 @@ const withoutNullAndUndefinedValues = (o: {}) => ([, v]) => v !== null && v !== undefined, )); - /** * Update the public magic metadata associated with a file on remote. * From 7d36808bb594e964dd915c4ea9e88c29aba83440 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 14:28:51 +0530 Subject: [PATCH 024/211] Sprinkled with TODOs, but a checkpoint --- web/packages/media/file-metadata.ts | 100 +++++++++++++++++++------- web/packages/new/photos/types/file.ts | 10 +++ web/packages/new/photos/utils/file.ts | 28 ++++---- 3 files changed, 100 insertions(+), 38 deletions(-) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index a2205e991c..8388734eae 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -2,9 +2,10 @@ import { encryptMetadata, type decryptMetadata } from "@/base/crypto/ente"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; import { type EnteFile } from "@/new/photos/types/file"; +import { mergeMetadata1 } from "@/new/photos/utils/file"; +import { ensure } from "@/utils/ensure"; +import { z } from "zod"; import { FileType } from "./file-type"; -import { z} from 'zod'; -import { nullToUndefined } from "@/utils/transform"; /** * Information about the file that never changes post upload. @@ -181,17 +182,19 @@ export interface PublicMagicMetadata { * might be other, newer, clients out there adding fields that the current * client might not we aware of, and we don't want to overwrite them. */ -const PublicMagicMetadata = z.object({ - // [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] - // - // Using `optional` is accurate here. The key is optional, but the value - // itself is not optional. Zod doesn't work with - // `exactOptionalPropertyTypes` yet, but it seems to be on the roadmap so we - // suppress these mismatches. - // - // See: https://github.com/colinhacks/zod/issues/635#issuecomment-2196579063 - editedTime: z.number().optional(), -}).passthrough(); +const PublicMagicMetadata = z + .object({ + // [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] + // + // Using `optional` is accurate here. The key is optional, but the value + // itself is not optional. Zod doesn't work with + // `exactOptionalPropertyTypes` yet, but it seems to be on the roadmap so we + // suppress these mismatches. + // + // See: https://github.com/colinhacks/zod/issues/635#issuecomment-2196579063 + editedTime: z.number().optional(), + }) + .passthrough(); /** * A function that can be used to encrypt the contents of a metadata field @@ -223,8 +226,13 @@ export type DecryptMetadataF = typeof decryptMetadata; * If the file doesn't have any public magic metadata attached to it, return * `undefined`. */ -export const decryptPublicMagicMetadata = async (enteFile: EnteFile, decryptMetadataF: DecryptMetadataF): Promise => { +export const decryptPublicMagicMetadata = async ( + enteFile: EnteFile, + decryptMetadataF: DecryptMetadataF, +): Promise => { const envelope = enteFile.pubMagicMetadata; + // TODO: The underlying types need auditing. + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (!envelope) return undefined; // TODO: This function can be optimized to directly return the cached value @@ -233,20 +241,30 @@ export const decryptPublicMagicMetadata = async (enteFile: EnteFile, decryptMeta // data there, so that it is in a known good state (currently we exist in // parallel with other functions that do the similar things). - const jsonValue = typeof envelope.data == "string" ? decryptMetadataF(envelope.data, envelope.header, enteFile.key) : envelope.data; - const result = PublicMagicMetadata.parse(withoutNullAndUndefinedValues(jsonValue)); + const jsonValue = + typeof envelope.data == "string" + ? await decryptMetadataF( + envelope.data, + envelope.header, + enteFile.key, + ) + : envelope.data; + const result = PublicMagicMetadata.parse( + // TODO: Can we avoid this cast? + withoutNullAndUndefinedValues(jsonValue as object), + ); // @ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] envelope.data = result; // @ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] return result; -} +}; -const withoutNullAndUndefinedValues = (o: {}) => - Object.fromEntries(Object.entries(o).filter( - ([, v]) => v !== null && v !== undefined, - )); +const withoutNullAndUndefinedValues = (o: object) => + Object.fromEntries( + Object.entries(o).filter(([, v]) => v !== null && v !== undefined), + ); /** * Update the public magic metadata associated with a file on remote. @@ -263,7 +281,11 @@ const withoutNullAndUndefinedValues = (o: {}) => * @param metadataUpdates A subset of {@link PublicMagicMetadata} containing the * fields that we want to add or update. * - * @param encryptMetadataF A function that is used to encrypt the metadata. + * @param encryptMetadataF A function that is used to encrypt the updated + * metadata. + * + * @param decryptMetadataF A function that is used to decrypt the existing + * metadata. * * @returns A {@link EnteFile} object with the updated public magic metadata. */ @@ -271,10 +293,38 @@ export const updateRemotePublicMagicMetadata = async ( enteFile: EnteFile, metadataUpdates: Partial, encryptMetadataF: EncryptMetadataF, + decryptMetadataF: DecryptMetadataF, ) => { - const updatedMetadata = { - ...file. - } + const existingMetadata = await decryptPublicMagicMetadata( + enteFile, + decryptMetadataF, + ); + + const updatedMetadata = { ...(existingMetadata ?? {}), ...metadataUpdates }; + + // The underlying types of enteFile.pubMagicMetadata are incorrect + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + const metadataVersion = enteFile.pubMagicMetadata?.version ?? 1; + + const updateRequest = await updateMagicMetadataRequest( + enteFile, + updatedMetadata, + metadataVersion, + encryptMetadataF, + ); + + const updatedEnvelope = ensure(updateRequest.metadataList[0]).magicMetadata; + + await putFilesMagicMetadata(updateRequest); + + // Modify the in-memory object. + // + // TODO: This is hacky, and we should find a better way, I'm just retaining + // the existing behaviour. Also, we need a cast since the underlying + // pubMagicMetadata type is imprecise. + enteFile.pubMagicMetadata = + updatedEnvelope as typeof enteFile.pubMagicMetadata; + return mergeMetadata1(enteFile); }; /** diff --git a/web/packages/new/photos/types/file.ts b/web/packages/new/photos/types/file.ts index aff511a337..c07c79649f 100644 --- a/web/packages/new/photos/types/file.ts +++ b/web/packages/new/photos/types/file.ts @@ -46,8 +46,18 @@ export interface EnteFile > { metadata: Metadata; magicMetadata: FileMagicMetadata; + /** + * The envelope containing the public magic metadata associated with this + * file. + */ pubMagicMetadata: FilePublicMagicMetadata; isTrashed?: boolean; + /** + * The base64 encoded encryption key associated with this file. + * + * This key is used to encrypt both the file's contents, and any associated + * data (e.g., metadatum, thumbnail) for the file. + */ key: string; src?: string; srcURLs?: SourceURLs; diff --git a/web/packages/new/photos/utils/file.ts b/web/packages/new/photos/utils/file.ts index f63d775319..3b9281e468 100644 --- a/web/packages/new/photos/utils/file.ts +++ b/web/packages/new/photos/utils/file.ts @@ -39,20 +39,22 @@ export const fileLogID = (enteFile: EnteFile) => * its filename. */ export function mergeMetadata(files: EnteFile[]): EnteFile[] { - return files.map((file) => { - // TODO: Until the types reflect reality - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (file.pubMagicMetadata?.data.editedTime) { - file.metadata.creationTime = file.pubMagicMetadata.data.editedTime; - } - // TODO: Until the types reflect reality - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - if (file.pubMagicMetadata?.data.editedName) { - file.metadata.title = file.pubMagicMetadata.data.editedName; - } + return files.map((file) => mergeMetadata1(file)); +} + +export function mergeMetadata1(file: EnteFile): EnteFile { + // TODO: Until the types reflect reality + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (file.pubMagicMetadata?.data.editedTime) { + file.metadata.creationTime = file.pubMagicMetadata.data.editedTime; + } + // TODO: Until the types reflect reality + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (file.pubMagicMetadata?.data.editedName) { + file.metadata.title = file.pubMagicMetadata.data.editedName; + } - return file; - }); + return file; } /** From 07786140f9d3363948b403014bb8f63e0b82a0be Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 14:44:44 +0530 Subject: [PATCH 025/211] Give it a spin --- .../FileInfo/RenderCreationTime.tsx | 21 ++++++++++++---- web/packages/media/file-metadata.ts | 24 +++++++++---------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx index 636ecc8b8b..e5a5db85ca 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx @@ -1,15 +1,15 @@ import log from "@/base/log"; -import type { ParsedMetadataDate } from "@/media/file-metadata"; +import { + updateRemotePublicMagicMetadata, + type ParsedMetadataDate, +} from "@/media/file-metadata"; import { PhotoDateTimePicker } from "@/new/photos/components/PhotoDateTimePicker"; import { EnteFile } from "@/new/photos/types/file"; import { FlexWrapper } from "@ente/shared/components/Container"; import { formatDate, formatTime } from "@ente/shared/time/format"; import CalendarTodayIcon from "@mui/icons-material/CalendarToday"; import { useState } from "react"; -import { - changeFileCreationTime, - updateExistingFilePubMetadata, -} from "utils/file"; +import ComlinkCryptoWorker from "@ente/shared/crypto"; import InfoItem from "./InfoItem"; export function RenderCreationTime({ @@ -37,11 +37,22 @@ export function RenderCreationTime({ closeEditMode(); return; } + const editedTime = unixTimeInMicroSec; + const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + + /* TODO(MR): Exif const updatedFile = await changeFileCreationTime( file, unixTimeInMicroSec, ); updateExistingFilePubMetadata(file, updatedFile); + */ + updateRemotePublicMagicMetadata( + file, + { editedTime }, + cryptoWorker.encryptMetadata, + cryptoWorker.decryptMetadata, + ); scheduleUpdate(); } } catch (e) { diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 8388734eae..af4907d05f 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -269,11 +269,11 @@ const withoutNullAndUndefinedValues = (o: object) => /** * Update the public magic metadata associated with a file on remote. * - * This function updates the public magic metadata on remote, and also returns a - * new {@link EnteFile} object with the updated values, but it does not update - * the state of the local databases. The caller needs to ensure that we - * subsequently sync with remote to fetch the updates as part of the diff and - * update the {@link EnteFile} that is persisted in our local db. + * This function updates the public magic metadata on remote, and also modifies + * the provided {@link EnteFile} object with the updated values in place, but it + * does not update the state of the local databases. The caller needs to ensure + * that we subsequently sync with remote to fetch the updates as part of the + * diff and update the {@link EnteFile} that is persisted in our local db. * * @param enteFile The {@link EnteFile} whose public magic metadata we want to * update. @@ -286,8 +286,6 @@ const withoutNullAndUndefinedValues = (o: object) => * * @param decryptMetadataF A function that is used to decrypt the existing * metadata. - * - * @returns A {@link EnteFile} object with the updated public magic metadata. */ export const updateRemotePublicMagicMetadata = async ( enteFile: EnteFile, @@ -315,16 +313,16 @@ export const updateRemotePublicMagicMetadata = async ( const updatedEnvelope = ensure(updateRequest.metadataList[0]).magicMetadata; - await putFilesMagicMetadata(updateRequest); + await putFilesPublicMagicMetadata(updateRequest); - // Modify the in-memory object. + // Modify the in-memory object. TODO: This is hacky, and we should find a + // better way, I'm just retaining the existing behaviour. // - // TODO: This is hacky, and we should find a better way, I'm just retaining - // the existing behaviour. Also, we need a cast since the underlying - // pubMagicMetadata type is imprecise. + // Also, we need a cast since the underlying pubMagicMetadata type is + // imprecise. enteFile.pubMagicMetadata = updatedEnvelope as typeof enteFile.pubMagicMetadata; - return mergeMetadata1(enteFile); + mergeMetadata1(enteFile); }; /** From 2c6298d6c3aeeaff63b000131d318e176b425e91 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 14:52:52 +0530 Subject: [PATCH 026/211] Test 2 --- .../FileInfo/RenderCreationTime.tsx | 45 ++++++++++++------- web/packages/media/file-metadata.ts | 3 ++ 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx index e5a5db85ca..3b7d135705 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx @@ -6,10 +6,14 @@ import { import { PhotoDateTimePicker } from "@/new/photos/components/PhotoDateTimePicker"; import { EnteFile } from "@/new/photos/types/file"; import { FlexWrapper } from "@ente/shared/components/Container"; +import ComlinkCryptoWorker from "@ente/shared/crypto"; import { formatDate, formatTime } from "@ente/shared/time/format"; import CalendarTodayIcon from "@mui/icons-material/CalendarToday"; import { useState } from "react"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; +import { + changeFileCreationTime, + updateExistingFilePubMetadata, +} from "utils/file"; import InfoItem from "./InfoItem"; export function RenderCreationTime({ @@ -38,21 +42,32 @@ export function RenderCreationTime({ return; } const editedTime = unixTimeInMicroSec; - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); - /* TODO(MR): Exif - const updatedFile = await changeFileCreationTime( - file, - unixTimeInMicroSec, - ); - updateExistingFilePubMetadata(file, updatedFile); - */ - updateRemotePublicMagicMetadata( - file, - { editedTime }, - cryptoWorker.encryptMetadata, - cryptoWorker.decryptMetadata, - ); + log.debug(() => ["before", file.pubMagicMetadata]); + + /* TODO(MR): Exif */ + // eslint-disable-next-line no-constant-condition + if (true) { + const updatedFile = await changeFileCreationTime( + file, + editedTime, + ); + updateExistingFilePubMetadata(file, updatedFile); + } + // eslint-disable-next-line no-constant-condition + if (false) { + const cryptoWorker = + await ComlinkCryptoWorker.getInstance(); + updateRemotePublicMagicMetadata( + file, + { editedTime }, + cryptoWorker.encryptMetadata, + cryptoWorker.decryptMetadata, + ); + } + + log.debug(() => ["after", file.pubMagicMetadata]); + scheduleUpdate(); } } catch (e) { diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index af4907d05f..23aea1ed97 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -322,6 +322,9 @@ export const updateRemotePublicMagicMetadata = async ( // imprecise. enteFile.pubMagicMetadata = updatedEnvelope as typeof enteFile.pubMagicMetadata; + // If the above is hacky, this is even worse. TODO, or at least move to a + // more visible place. + enteFile.pubMagicMetadata.version = enteFile.pubMagicMetadata.version + 1; mergeMetadata1(enteFile); }; From d30773e68a89e200acb532ba5df4cc48ccf84e08 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 15:22:38 +0530 Subject: [PATCH 027/211] Fix --- .../src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx index 3b7d135705..0bba0c6897 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx @@ -58,7 +58,7 @@ export function RenderCreationTime({ if (false) { const cryptoWorker = await ComlinkCryptoWorker.getInstance(); - updateRemotePublicMagicMetadata( + await updateRemotePublicMagicMetadata( file, { editedTime }, cryptoWorker.encryptMetadata, From 962a260e4b6e4afb726aadba05c79ae51f30ed17 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 15:27:05 +0530 Subject: [PATCH 028/211] Copy over another hack --- .../components/PhotoViewer/FileInfo/RenderCreationTime.tsx | 4 ++-- web/packages/media/file-metadata.ts | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx index 0bba0c6897..ea092d77ae 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx @@ -47,7 +47,7 @@ export function RenderCreationTime({ /* TODO(MR): Exif */ // eslint-disable-next-line no-constant-condition - if (true) { + if (false) { const updatedFile = await changeFileCreationTime( file, editedTime, @@ -55,7 +55,7 @@ export function RenderCreationTime({ updateExistingFilePubMetadata(file, updatedFile); } // eslint-disable-next-line no-constant-condition - if (false) { + if (true) { const cryptoWorker = await ComlinkCryptoWorker.getInstance(); await updateRemotePublicMagicMetadata( diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 23aea1ed97..be3ac2e3c9 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -325,6 +325,7 @@ export const updateRemotePublicMagicMetadata = async ( // If the above is hacky, this is even worse. TODO, or at least move to a // more visible place. enteFile.pubMagicMetadata.version = enteFile.pubMagicMetadata.version + 1; + enteFile.pubMagicMetadata.data = updatedMetadata; mergeMetadata1(enteFile); }; From 9ceae94071a4fc5b20c1a05518c5a144e77c7875 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 15:30:38 +0530 Subject: [PATCH 029/211] Update docs --- web/packages/media/file-metadata.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index be3ac2e3c9..bcb8216c7b 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -343,8 +343,8 @@ interface RemoteMagicMetadata { /** * Monotonically increasing iteration of this metadata object. * - * The version starts at 1. Each time a client updates the underlying magic - * metadata JSONs for a file, it increments this version number. + * The version starts at 1. Remote increments this version number each time + * a client updates the corresponding magic metadata field for the file. */ version: number; /** From ec91e75780d1527fa365135c96b90fd237a1dea2 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:29:10 +0530 Subject: [PATCH 030/211] [server] Handle fileData cleanup on file Deletion --- server/cmd/museum/main.go | 14 ++- server/ente/filedata/path.go | 12 +- ...e.down.sql => 89_file_data_table.down.sql} | 0 ...table.up.sql => 89_file_data_table.up.sql} | 2 + server/pkg/api/file_data.go | 5 + server/pkg/controller/file.go | 2 - server/pkg/controller/filedata/delete.go | 119 ++++++++++++++++++ server/pkg/controller/filedata/file_object.go | 5 - server/pkg/repo/filedata/repository.go | 82 ++++++++++++ server/pkg/repo/object.go | 9 ++ server/pkg/repo/queue.go | 2 + 11 files changed, 236 insertions(+), 16 deletions(-) rename server/migrations/{89_derived_data_table.down.sql => 89_file_data_table.down.sql} (100%) rename server/migrations/{89_derived_data_table.up.sql => 89_file_data_table.up.sql} (96%) create mode 100644 server/pkg/controller/filedata/delete.go delete mode 100644 server/pkg/controller/filedata/file_object.go diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index c56045e4a4..602342c584 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -6,6 +6,7 @@ import ( b64 "encoding/base64" "fmt" "github.com/ente-io/museum/pkg/controller/file_copy" + "github.com/ente-io/museum/pkg/controller/filedata" "net/http" "os" "os/signal" @@ -49,6 +50,7 @@ import ( castRepo "github.com/ente-io/museum/pkg/repo/cast" "github.com/ente-io/museum/pkg/repo/datacleanup" "github.com/ente-io/museum/pkg/repo/embedding" + fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata" "github.com/ente-io/museum/pkg/repo/kex" "github.com/ente-io/museum/pkg/repo/passkey" "github.com/ente-io/museum/pkg/repo/remotestore" @@ -163,6 +165,7 @@ func main() { fileRepo := &repo.FileRepository{DB: db, S3Config: s3Config, QueueRepo: queueRepo, ObjectRepo: objectRepo, ObjectCleanupRepo: objectCleanupRepo, ObjectCopiesRepo: objectCopiesRepo, UsageRepo: usageRepo} + fileDataRepo := &fileDataRepo.Repository{DB: db} familyRepo := &repo.FamilyRepository{DB: db} trashRepo := &repo.TrashRepository{DB: db, ObjectRepo: objectRepo, FileRepo: fileRepo, QueueRepo: queueRepo} publicCollectionRepo := repo.NewPublicCollectionRepository(db, viper.GetString("apps.public-albums")) @@ -239,6 +242,9 @@ func main() { FileRepo: fileRepo, } + accessCtrl := access.NewAccessController(collectionRepo, fileRepo) + fileDataCtrl := filedata.New(fileDataRepo, accessCtrl, objectCleanupController, s3Config, queueRepo, taskLockingRepo, fileRepo, collectionRepo, hostName) + fileController := &controller.FileController{ FileRepo: fileRepo, ObjectRepo: objectRepo, @@ -288,8 +294,6 @@ func main() { JwtSecret: jwtSecretBytes, } - accessCtrl := access.NewAccessController(collectionRepo, fileRepo) - collectionController := &controller.CollectionController{ CollectionRepo: collectionRepo, AccessCtrl: accessCtrl, @@ -402,6 +406,7 @@ func main() { fileHandler := &api.FileHandler{ Controller: fileController, FileCopyCtrl: fileCopyCtrl, + FileDataCtrl: fileDataCtrl, } privateAPI.GET("/files/upload-urls", fileHandler.GetUploadURLs) privateAPI.GET("/files/multipart-upload-urls", fileHandler.GetMultipartUploadURLs) @@ -707,7 +712,7 @@ func main() { setupAndStartCrons( userAuthRepo, publicCollectionRepo, twoFactorRepo, passkeysRepo, fileController, taskLockingRepo, emailNotificationCtrl, trashController, pushController, objectController, dataCleanupController, storageBonusCtrl, - embeddingController, healthCheckHandler, kexCtrl, castDb) + embeddingController, fileDataCtrl, healthCheckHandler, kexCtrl, castDb) // Create a new collector, the name will be used as a label on the metrics collector := sqlstats.NewStatsCollector("prod_db", db) @@ -836,6 +841,7 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, publicCollectionR dataCleanupCtrl *dataCleanupCtrl.DeleteUserCleanupController, storageBonusCtrl *storagebonus.Controller, embeddingCtrl *embeddingCtrl.Controller, + fileDataCtrl *filedata.Controller, healthCheckHandler *api.HealthCheckHandler, kexCtrl *kexCtrl.Controller, castDb castRepo.Repository) { @@ -879,8 +885,10 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, publicCollectionR schedule(c, "@every 2m", func() { fileController.CleanupDeletedFiles() }) + fileDataCtrl.CleanUpDeletedFileData() schedule(c, "@every 101s", func() { embeddingCtrl.CleanupDeletedEmbeddings() + fileDataCtrl.CleanUpDeletedFileData() }) schedule(c, "@every 10m", func() { diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.go index c656fce164..5dd4f616ef 100644 --- a/server/ente/filedata/path.go +++ b/server/ente/filedata/path.go @@ -5,13 +5,13 @@ import ( "github.com/ente-io/museum/ente" ) -// basePrefix returns the base prefix for all objects related to a file. To check if the file data is deleted, +// BasePrefix returns the base prefix for all objects related to a file. To check if the file data is deleted, // ensure that there's no file in the S3 bucket with this prefix. -func basePrefix(fileID int64, ownerID int64) string { +func BasePrefix(fileID int64, ownerID int64) string { return fmt.Sprintf("%d/file-data/%d/", ownerID, fileID) } -func allObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string { +func AllObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string { switch oType { case ente.PreviewVideo: return []string{previewVideoPath(fileID, ownerID), previewVideoPlaylist(fileID, ownerID)} @@ -26,7 +26,7 @@ func allObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string { } func previewVideoPath(fileID int64, ownerID int64) string { - return fmt.Sprintf("%s%s", basePrefix(fileID, ownerID), string(ente.PreviewVideo)) + return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.PreviewVideo)) } func previewVideoPlaylist(fileID int64, ownerID int64) string { @@ -34,9 +34,9 @@ func previewVideoPlaylist(fileID int64, ownerID int64) string { } func previewImagePath(fileID int64, ownerID int64) string { - return fmt.Sprintf("%s%s", basePrefix(fileID, ownerID), string(ente.PreviewImage)) + return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.PreviewImage)) } func derivedMetaPath(fileID int64, ownerID int64) string { - return fmt.Sprintf("%s%s", basePrefix(fileID, ownerID), string(ente.DerivedMeta)) + return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.DerivedMeta)) } diff --git a/server/migrations/89_derived_data_table.down.sql b/server/migrations/89_file_data_table.down.sql similarity index 100% rename from server/migrations/89_derived_data_table.down.sql rename to server/migrations/89_file_data_table.down.sql diff --git a/server/migrations/89_derived_data_table.up.sql b/server/migrations/89_file_data_table.up.sql similarity index 96% rename from server/migrations/89_derived_data_table.up.sql rename to server/migrations/89_file_data_table.up.sql index 53b0ae95f3..3012cea296 100644 --- a/server/migrations/89_derived_data_table.up.sql +++ b/server/migrations/89_file_data_table.up.sql @@ -1,3 +1,5 @@ +ALTER TYPE OBJECT_TYPE ADD VALUE 'derivedMeta'; +ALTER TYPE s3region ADD VALUE 'b5'; -- Create the derived table CREATE TABLE IF NOT EXISTS file_data ( diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go index 81546b52ef..73ddeb6f27 100644 --- a/server/pkg/api/file_data.go +++ b/server/pkg/api/file_data.go @@ -18,6 +18,11 @@ func (f *FileHandler) PutFileData(ctx *gin.Context) { ctx.JSON(http.StatusBadRequest, err) return } + reqInt := &req + if reqInt.Version == nil { + version := 1 + reqInt.Version = &version + } err := f.FileDataCtrl.InsertOrUpdate(ctx, &req) if err != nil { handler.Error(ctx, err) diff --git a/server/pkg/controller/file.go b/server/pkg/controller/file.go index 23d9c6e8f9..c001089d4d 100644 --- a/server/pkg/controller/file.go +++ b/server/pkg/controller/file.go @@ -284,8 +284,6 @@ func (c *FileController) GetUploadURLs(ctx context.Context, userID int64, count return urls, nil } - - // GetFileURL verifies permissions and returns a presigned url to the requested file func (c *FileController) GetFileURL(ctx *gin.Context, userID int64, fileID int64) (string, error) { err := c.verifyFileAccess(userID, fileID) diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go new file mode 100644 index 0000000000..e9ada5a0f8 --- /dev/null +++ b/server/pkg/controller/filedata/delete.go @@ -0,0 +1,119 @@ +package filedata + +import ( + "context" + "fmt" + "github.com/ente-io/museum/ente/filedata" + "github.com/ente-io/museum/pkg/repo" + "github.com/ente-io/museum/pkg/utils/time" + log "github.com/sirupsen/logrus" + "strconv" +) + +// CleanUpDeletedFileData clears associated file data from the object store +func (c *Controller) CleanUpDeletedFileData() { + log.Info("Cleaning up deleted file data") + if c.cleanupCronRunning { + log.Info("Skipping CleanUpDeletedFileData cron run as another instance is still running") + return + } + c.cleanupCronRunning = true + defer func() { + c.cleanupCronRunning = false + }() + items, err := c.QueueRepo.GetItemsReadyForDeletion(repo.DeleteFileDataQueue, 200) + if err != nil { + log.WithError(err).Error("Failed to fetch items from queue") + return + } + for _, i := range items { + c.deleteFileData(i) + } +} + +func (c *Controller) deleteFileData(qItem repo.QueueItem) { + lockName := fmt.Sprintf("FileDataDelete:%s", qItem.Item) + lockStatus, err := c.TaskLockingRepo.AcquireLock(lockName, time.MicrosecondsAfterHours(1), c.HostName) + ctxLogger := log.WithField("item", qItem.Item).WithField("queue_id", qItem.Id) + if err != nil || !lockStatus { + ctxLogger.Warn("unable to acquire lock") + return + } + defer func() { + err = c.TaskLockingRepo.ReleaseLock(lockName) + if err != nil { + ctxLogger.Errorf("Error while releasing lock %s", err) + } + }() + ctxLogger.Debug("Deleting all file data") + fileID, _ := strconv.ParseInt(qItem.Item, 10, 64) + ownerID, err := c.FileRepo.GetOwnerID(fileID) + if err != nil { + ctxLogger.WithError(err).Error("Failed to fetch ownerID") + return + } + rows, err := c.Repo.GetFileData(context.Background(), fileID) + if err != nil { + ctxLogger.WithError(err).Error("Failed to fetch datacenters") + return + } + for i := range rows { + fileDataRow := rows[i] + objectKeys := filedata.AllObjects(fileID, ownerID, fileDataRow.Type) + // Delete from delete/stale buckets + for j := range fileDataRow.DeleteFromBuckets { + bucketID := fileDataRow.DeleteFromBuckets[j] + for k := range objectKeys { + err = c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKeys[k], bucketID) + if err != nil { + ctxLogger.WithError(err).Error("Failed to delete object from datacenter") + return + } + } + dbErr := c.Repo.RemoveBucketFromDeletedBuckets(fileDataRow, bucketID) + if dbErr != nil { + ctxLogger.WithError(dbErr).Error("Failed to remove from db") + return + } + } + // Delete from replicated buckets + for j := range fileDataRow.ReplicatedBuckets { + bucketID := fileDataRow.ReplicatedBuckets[j] + for k := range objectKeys { + err = c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKeys[k], bucketID) + if err != nil { + ctxLogger.WithError(err).Error("Failed to delete object from datacenter") + return + } + } + dbErr := c.Repo.RemoveBucketFromReplicatedBuckets(fileDataRow, bucketID) + if dbErr != nil { + ctxLogger.WithError(dbErr).Error("Failed to remove from db") + return + } + } + // Delete from Latest bucket + for k := range objectKeys { + err = c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKeys[k], fileDataRow.LatestBucket) + if err != nil { + ctxLogger.WithError(err).Error("Failed to delete object from datacenter") + return + } + } + dbErr := c.Repo.DeleteFileData(context.Background(), fileDataRow.FileID, fileDataRow.Type, fileDataRow.LatestBucket) + if dbErr != nil { + ctxLogger.WithError(dbErr).Error("Failed to remove from db") + return + } + } + if err != nil { + ctxLogger.WithError(err).Error("Failed delete data") + return + } + err = c.QueueRepo.DeleteItem(repo.DeleteFileDataQueue, qItem.Item) + if err != nil { + ctxLogger.WithError(err).Error("Failed to remove item from the queue") + return + } + ctxLogger.Info("Successfully deleted all file data") +} diff --git a/server/pkg/controller/filedata/file_object.go b/server/pkg/controller/filedata/file_object.go deleted file mode 100644 index aa627fb82c..0000000000 --- a/server/pkg/controller/filedata/file_object.go +++ /dev/null @@ -1,5 +0,0 @@ -package filedata - -func (c *Controller) f() { - -} diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go index 02d7ccb57b..8b56a76d33 100644 --- a/server/pkg/repo/filedata/repository.go +++ b/server/pkg/repo/filedata/repository.go @@ -52,6 +52,88 @@ func (r *Repository) GetFilesData(ctx context.Context, oType ente.ObjectType, fi return nil, stacktrace.Propagate(err, "") } return convertRowsToFilesData(rows) +} + +func (r *Repository) GetFileData(ctx context.Context, fileIDs int64) ([]filedata.Row, error) { + rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at + FROM file_data + WHERE file_id = $1`, fileIDs) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return convertRowsToFilesData(rows) +} + +func (r *Repository) RemoveBucketFromDeletedBuckets(row filedata.Row, bucketID string) error { + query := ` + UPDATE file_data + SET delete_from_buckets = array( + SELECT DISTINCT elem FROM unnest( + array_remove( + file_data.delete_from_buckets, + $1 + ) + ) AS elem + WHERE elem IS NOT NULL + ) + WHERE file_id = $2 AND data_type = $3 and is_deleted = true` + result, err := r.DB.Exec(query, bucketID, row.FileID, string(row.Type)) + if err != nil { + return stacktrace.Propagate(err, "failed to remove bucket from deleted buckets") + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return stacktrace.Propagate(err, "") + } + if rowsAffected == 0 { + return stacktrace.NewError("bucket not removed from deleted buckets") + } + return nil +} + +func (r *Repository) RemoveBucketFromReplicatedBuckets(row filedata.Row, bucketID string) error { + query := ` + UPDATE file_data + SET replicated_buckets = array( + SELECT DISTINCT elem FROM unnest( + array_remove( + file_data.replicated_buckets, + $1 + ) + ) AS elem + WHERE elem IS NOT NULL + ) + WHERE file_id = $2 AND data_type = $3` + result, err := r.DB.Exec(query, bucketID, row.FileID, string(row.Type)) + if err != nil { + return stacktrace.Propagate(err, "failed to remove bucket from replicated buckets") + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return stacktrace.Propagate(err, "") + } + if rowsAffected == 0 { + return stacktrace.NewError("bucket not removed from deleted buckets") + } + return nil +} + +func (r *Repository) DeleteFileData(ctx context.Context, fileID int64, oType ente.ObjectType, latestBucketID string) error { + query := ` +DELETE FROM file_data +WHERE file_id = $1 AND data_type = $2 AND latest_bucket = $3 AND replicated_buckets = ARRAY[]::s3region[] AND delete_from_buckets = ARRAY[]::s3region[]` + res, err := r.DB.ExecContext(ctx, query, fileID, string(oType), latestBucketID) + if err != nil { + return stacktrace.Propagate(err, "") + } + rowsAffected, err := res.RowsAffected() + if err != nil { + return stacktrace.Propagate(err, "") + } + if rowsAffected == 0 { + return stacktrace.NewError("file data not deleted") + } + return nil } diff --git a/server/pkg/repo/object.go b/server/pkg/repo/object.go index 052278402d..fc02e2d25c 100644 --- a/server/pkg/repo/object.go +++ b/server/pkg/repo/object.go @@ -148,12 +148,21 @@ func (repo *ObjectRepository) MarkObjectsAsDeletedForFileIDs(ctx context.Context for _, fileID := range fileIDs { embeddingsToBeDeleted = append(embeddingsToBeDeleted, strconv.FormatInt(fileID, 10)) } + _, err = tx.ExecContext(ctx, `UPDATE file_data SET is_deleted = TRUE WHERE file_id = ANY($1)`, pq.Array(fileIDs)) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } err = repo.QueueRepo.AddItems(ctx, tx, DeleteEmbeddingsQueue, embeddingsToBeDeleted) if err != nil { return nil, stacktrace.Propagate(err, "") } + err = repo.QueueRepo.AddItems(ctx, tx, DeleteFileDataQueue, embeddingsToBeDeleted) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + _, err = tx.ExecContext(ctx, `UPDATE object_keys SET is_deleted = TRUE WHERE file_id = ANY($1)`, pq.Array(fileIDs)) if err != nil { return nil, stacktrace.Propagate(err, "") diff --git a/server/pkg/repo/queue.go b/server/pkg/repo/queue.go index 49544dbc8c..e4800aea9c 100644 --- a/server/pkg/repo/queue.go +++ b/server/pkg/repo/queue.go @@ -23,6 +23,7 @@ var itemDeletionDelayInMinMap = map[string]int64{ DropFileEncMedataQueue: -1 * 24 * 60, // -ve value to ensure attributes are immediately removed DeleteObjectQueue: 45 * 24 * 60, // 45 days in minutes DeleteEmbeddingsQueue: -1 * 24 * 60, // -ve value to ensure embeddings are immediately removed + DeleteFileDataQueue: -1 * 24 * 60, // -ve value to ensure file-data is immediately removed TrashCollectionQueueV3: -1 * 24 * 60, // -ve value to ensure collections are immediately marked as trashed TrashEmptyQueue: -1 * 24 * 60, // -ve value to ensure empty trash request are processed in next cron run RemoveComplianceHoldQueue: -1 * 24 * 60, // -ve value to ensure compliance hold is removed in next cron run @@ -32,6 +33,7 @@ const ( DropFileEncMedataQueue string = "dropFileEncMetata" DeleteObjectQueue string = "deleteObject" DeleteEmbeddingsQueue string = "deleteEmbedding" + DeleteFileDataQueue string = "deleteFileData" OutdatedObjectsQueue string = "outdatedObject" // Deprecated: Keeping it till we clean up items from the queue DB. TrashCollectionQueue string = "trashCollection" From 84fa8f343b443da733a7e97108204a45c52bfc48 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 6 Aug 2024 15:34:27 +0530 Subject: [PATCH 031/211] clean up --- server/cmd/museum/main.go | 2 - server/pkg/utils/s3config/s3config.go | 58 --------------------------- 2 files changed, 60 deletions(-) diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index 602342c584..10b5a7fc6c 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -136,8 +136,6 @@ func main() { }, []string{"method"}) s3Config := s3config.NewS3Config() - //s3Config.EmptyB5Bucket(1580559962386453) - passkeysRepo, err := passkey.NewRepository(db) if err != nil { panic(err) diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index b0f88ee958..b6ec9ba2af 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -217,64 +217,6 @@ func (config *S3Config) GetDerivedStorageBucket() *string { return config.GetBucket(config.derivedStorageDC) } -func (config *S3Config) EmptyB5Bucket(userID int64) { - bucket := config.GetBucket("b5") - prefix := fmt.Sprintf("%d/", userID) - // Create an S3 client - s3Client := config.GetS3Client("b5") - - // List objects with the specified prefix - listObjectsInput := &s3.ListObjectsV2Input{ - Bucket: aws.String(*bucket), - Prefix: aws.String(prefix), - } - - for { - // Get the list of objects - listObjectsOutput, err := s3Client.ListObjectsV2(listObjectsInput) - if err != nil { - log.Fatalf("unable to list objects, %v", err) - } - - if len(listObjectsOutput.Contents) == 0 { - fmt.Println("No objects found with the specified prefix.") - break - } - - // Create a slice of object identifiers to delete - var objectIdentifiers []*s3.ObjectIdentifier - for _, object := range listObjectsOutput.Contents { - objectIdentifiers = append(objectIdentifiers, &s3.ObjectIdentifier{ - Key: object.Key, - }) - } - - // Delete the objects - deleteObjectsInput := &s3.DeleteObjectsInput{ - Bucket: aws.String(*bucket), - Delete: &s3.Delete{ - Objects: objectIdentifiers, - Quiet: aws.Bool(true), - }, - } - - _, err = s3Client.DeleteObjects(deleteObjectsInput) - if err != nil { - log.Fatalf("unable to delete objects, %v", err) - } - - fmt.Printf("Deleted %d objects.\n", len(objectIdentifiers)) - - // If there are more objects to list, set the continuation token for the next request - if *listObjectsOutput.IsTruncated { - listObjectsInput.ContinuationToken = listObjectsOutput.NextContinuationToken - } else { - break - } - } - fmt.Println("All objects with the specified prefix have been deleted.") -} - func (config *S3Config) GetDerivedStorageS3Client() *s3.S3 { s3Client := config.GetS3Client(config.derivedStorageDC) return &s3Client From af4064b97a3a43815ef67b4787b591c9ef8ca141 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 16:24:40 +0530 Subject: [PATCH 032/211] Prepare for merge --- .../FileInfo/RenderCreationTime.tsx | 4 ++-- web/packages/media/file-metadata.ts | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx index ea092d77ae..0bba0c6897 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx @@ -47,7 +47,7 @@ export function RenderCreationTime({ /* TODO(MR): Exif */ // eslint-disable-next-line no-constant-condition - if (false) { + if (true) { const updatedFile = await changeFileCreationTime( file, editedTime, @@ -55,7 +55,7 @@ export function RenderCreationTime({ updateExistingFilePubMetadata(file, updatedFile); } // eslint-disable-next-line no-constant-condition - if (true) { + if (false) { const cryptoWorker = await ComlinkCryptoWorker.getInstance(); await updateRemotePublicMagicMetadata( diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index bcb8216c7b..ae689797d9 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -137,6 +137,24 @@ export enum ItemVisibility { * with whom the file has been shared. * * For more details, see [Note: Metadatum]. + * + * --- + * + * [Note: Optional magic metadata keys] + * + * Remote does not support nullish (`undefined` or `null`) values for the keys + * in the magic metadata associated with a file. All of the keys themselves are + * optional though. + * + * That is, all magic metadata properties are of the form: + * + * foo?: T + * + * And never like: + * + * foo: T | undefined + * + * Also see: [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet]. */ export interface PublicMagicMetadata { /** @@ -398,6 +416,7 @@ export const updateMagicMetadataRequest = async ( encryptMetadataF: EncryptMetadataF, ): Promise => { // Drop all null or undefined values to obtain the syncable entries. + // See: [Note: Optional magic metadata keys]. const validEntries = Object.entries(metadata).filter( ([, v]) => v !== null && v !== undefined, ); From 98a6bf91648e052ae8e33b327cf2675b317e0511 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Tue, 6 Aug 2024 17:01:06 +0530 Subject: [PATCH 033/211] Store bucketID for temp objects --- server/ente/file.go | 2 +- server/migrations/89_file_data_table.up.sql | 1 + server/pkg/api/file.go | 48 ------------------- server/pkg/api/file_data.go | 52 +++++++++++++++++++++ server/pkg/controller/object_cleanup.go | 10 ++-- server/pkg/repo/object_cleanup.go | 12 +++-- 6 files changed, 68 insertions(+), 57 deletions(-) diff --git a/server/ente/file.go b/server/ente/file.go index 8a554b6b1b..1e00c7f258 100644 --- a/server/ente/file.go +++ b/server/ente/file.go @@ -202,7 +202,7 @@ type TempObject struct { ObjectKey string IsMultipart bool UploadID string - DataCenter string + BucketId string } // DuplicateFiles represents duplicate files diff --git a/server/migrations/89_file_data_table.up.sql b/server/migrations/89_file_data_table.up.sql index 3012cea296..49c9704ee6 100644 --- a/server/migrations/89_file_data_table.up.sql +++ b/server/migrations/89_file_data_table.up.sql @@ -1,3 +1,4 @@ +ALTER TABLE temp_objects ADD COLUMN IF NOT EXISTS bucket_id s3region; ALTER TYPE OBJECT_TYPE ADD VALUE 'derivedMeta'; ALTER TYPE s3region ADD VALUE 'b5'; -- Create the derived table diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go index 2d00b8eb53..2e15ade325 100644 --- a/server/pkg/api/file.go +++ b/server/pkg/api/file.go @@ -122,54 +122,6 @@ func (h *FileHandler) GetUploadURLs(c *gin.Context) { }) } -func (h *FileHandler) GetVideoUploadURL(c *gin.Context) { - enteApp := auth.GetApp(c) - userID, fileID := getUserAndFileIDs(c) - urls, err := h.Controller.GetVideoUploadUrl(c, userID, fileID, enteApp) - if err != nil { - handler.Error(c, stacktrace.Propagate(err, "")) - return - } - c.JSON(http.StatusOK, urls) -} - -func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) { - userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetPreviewUrl(c, userID, fileID) - if err != nil { - handler.Error(c, stacktrace.Propagate(err, "")) - return - } - c.JSON(http.StatusOK, gin.H{ - "url": url, - }) -} - -func (h *FileHandler) ReportVideoPlayList(c *gin.Context) { - var request ente.InsertOrUpdateEmbeddingRequest - if err := c.ShouldBindJSON(&request); err != nil { - handler.Error(c, - stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) - return - } - err := h.Controller.ReportVideoPreview(c, request) - if err != nil { - handler.Error(c, stacktrace.Propagate(err, "")) - return - } - c.Status(http.StatusOK) -} - -func (h *FileHandler) GetVideoPlaylist(c *gin.Context) { - fileID, _ := strconv.ParseInt(c.Param("fileID"), 10, 64) - response, err := h.Controller.GetPlaylist(c, fileID) - if err != nil { - handler.Error(c, stacktrace.Propagate(err, "")) - return - } - c.JSON(http.StatusOK, response) -} - // GetMultipartUploadURLs returns an array of PartUpload PresignedURLs func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) { enteApp := auth.GetApp(c) diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go index 73ddeb6f27..50015c3d4d 100644 --- a/server/pkg/api/file_data.go +++ b/server/pkg/api/file_data.go @@ -1,11 +1,15 @@ package api import ( + "fmt" "github.com/ente-io/museum/ente" fileData "github.com/ente-io/museum/ente/filedata" + "github.com/ente-io/museum/pkg/utils/auth" "github.com/ente-io/museum/pkg/utils/handler" + "github.com/ente-io/stacktrace" "github.com/gin-gonic/gin" "net/http" + "strconv" ) func (f *FileHandler) PutFileData(ctx *gin.Context) { @@ -45,3 +49,51 @@ func (f *FileHandler) GetFilesData(ctx *gin.Context) { } ctx.JSON(http.StatusOK, resp) } + +func (h *FileHandler) GetVideoUploadURL(c *gin.Context) { + enteApp := auth.GetApp(c) + userID, fileID := getUserAndFileIDs(c) + urls, err := h.Controller.GetVideoUploadUrl(c, userID, fileID, enteApp) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.JSON(http.StatusOK, urls) +} + +func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) { + userID, fileID := getUserAndFileIDs(c) + url, err := h.Controller.GetPreviewUrl(c, userID, fileID) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.JSON(http.StatusOK, gin.H{ + "url": url, + }) +} + +func (h *FileHandler) ReportVideoPlayList(c *gin.Context) { + var request ente.InsertOrUpdateEmbeddingRequest + if err := c.ShouldBindJSON(&request); err != nil { + handler.Error(c, + stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) + return + } + err := h.Controller.ReportVideoPreview(c, request) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.Status(http.StatusOK) +} + +func (h *FileHandler) GetVideoPlaylist(c *gin.Context) { + fileID, _ := strconv.ParseInt(c.Param("fileID"), 10, 64) + response, err := h.Controller.GetPlaylist(c, fileID) + if err != nil { + handler.Error(c, stacktrace.Propagate(err, "")) + return + } + c.JSON(http.StatusOK, response) +} diff --git a/server/pkg/controller/object_cleanup.go b/server/pkg/controller/object_cleanup.go index 91426cb56c..9a6a6055ce 100644 --- a/server/pkg/controller/object_cleanup.go +++ b/server/pkg/controller/object_cleanup.go @@ -166,8 +166,10 @@ func (c *ObjectCleanupController) removeUnreportedObjects() int { func (c *ObjectCleanupController) removeUnreportedObject(tx *sql.Tx, t ente.TempObject) error { // TODO: object_cleanup // This should use the DC from TempObject (once we start persisting it) - // dc := t.DataCenter - dc := c.S3Config.GetHotDataCenter() + dc := t.BucketId + if dc == "" { + dc = c.S3Config.GetHotDataCenter() + } logger := log.WithFields(log.Fields{ "task": "remove-unreported-objects", @@ -232,7 +234,7 @@ func (c *ObjectCleanupController) addCleanupEntryForObjectKey(objectKey string, err := c.Repo.AddTempObject(ente.TempObject{ ObjectKey: objectKey, IsMultipart: false, - DataCenter: dc, + BucketId: dc, }, expirationTime) return stacktrace.Propagate(err, "") } @@ -247,7 +249,7 @@ func (c *ObjectCleanupController) AddMultipartTempObjectKey(objectKey string, up ObjectKey: objectKey, IsMultipart: true, UploadID: uploadID, - DataCenter: dc, + BucketId: dc, }, expiry) return stacktrace.Propagate(err, "") } diff --git a/server/pkg/repo/object_cleanup.go b/server/pkg/repo/object_cleanup.go index 7074121381..b78910d052 100644 --- a/server/pkg/repo/object_cleanup.go +++ b/server/pkg/repo/object_cleanup.go @@ -25,8 +25,8 @@ type ObjectCleanupRepository struct { func (repo *ObjectCleanupRepository) AddTempObject(tempObject ente.TempObject, expirationTime int64) error { var err error if tempObject.IsMultipart { - _, err = repo.DB.Exec(`INSERT INTO temp_objects(object_key, expiration_time,upload_id,is_multipart) - VALUES($1, $2, $3, $4)`, tempObject.ObjectKey, expirationTime, tempObject.UploadID, tempObject.IsMultipart) + _, err = repo.DB.Exec(`INSERT INTO temp_objects(object_key, expiration_time,upload_id,is_multipart, bucket_id) + VALUES($1, $2, $3, $4)`, tempObject.ObjectKey, expirationTime, tempObject.UploadID, tempObject.IsMultipart, tempObject.BucketId) } else { _, err = repo.DB.Exec(`INSERT INTO temp_objects(object_key, expiration_time) VALUES($1, $2)`, tempObject.ObjectKey, expirationTime) @@ -62,7 +62,7 @@ func (repo *ObjectCleanupRepository) GetAndLockExpiredObjects() (*sql.Tx, []ente } rows, err := tx.Query(` - SELECT object_key, is_multipart, upload_id FROM temp_objects + SELECT object_key, is_multipart, upload_id, bucket_id FROM temp_objects WHERE expiration_time <= $1 LIMIT 1000 FOR UPDATE SKIP LOCKED @@ -83,7 +83,8 @@ func (repo *ObjectCleanupRepository) GetAndLockExpiredObjects() (*sql.Tx, []ente for rows.Next() { var tempObject ente.TempObject var uploadID sql.NullString - err := rows.Scan(&tempObject.ObjectKey, &tempObject.IsMultipart, &uploadID) + var bucketID sql.NullString + err := rows.Scan(&tempObject.ObjectKey, &tempObject.IsMultipart, &uploadID, &bucketID) if err != nil { rollback() return nil, nil, stacktrace.Propagate(err, "") @@ -91,6 +92,9 @@ func (repo *ObjectCleanupRepository) GetAndLockExpiredObjects() (*sql.Tx, []ente if tempObject.IsMultipart { tempObject.UploadID = uploadID.String } + if bucketID.Valid { + tempObject.BucketId = bucketID.String + } tempObjects = append(tempObjects, tempObject) } return tx, tempObjects, nil From a6cc6f24d080ce824b7d87c90c5fdf55db63b668 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 20:05:27 +0530 Subject: [PATCH 034/211] Use newer fields --- .../FileInfo/RenderCreationTime.tsx | 37 +++++-------------- web/packages/media/file-metadata.ts | 30 +++++++++++---- .../photos/components/PhotoDateTimePicker.tsx | 6 +-- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx index 0bba0c6897..d7b0a21e80 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx @@ -10,10 +10,6 @@ import ComlinkCryptoWorker from "@ente/shared/crypto"; import { formatDate, formatTime } from "@ente/shared/time/format"; import CalendarTodayIcon from "@mui/icons-material/CalendarToday"; import { useState } from "react"; -import { - changeFileCreationTime, - updateExistingFilePubMetadata, -} from "utils/file"; import InfoItem from "./InfoItem"; export function RenderCreationTime({ @@ -36,35 +32,22 @@ export function RenderCreationTime({ try { setLoading(true); if (isInEditMode && file) { - const unixTimeInMicroSec = pickedTime.timestamp; - if (unixTimeInMicroSec === file?.metadata.creationTime) { + const { dateTime, dateTimeOffset, timestamp } = pickedTime; + if (timestamp == file?.metadata.creationTime) { + // Same as before. closeEditMode(); return; } - const editedTime = unixTimeInMicroSec; log.debug(() => ["before", file.pubMagicMetadata]); - /* TODO(MR): Exif */ - // eslint-disable-next-line no-constant-condition - if (true) { - const updatedFile = await changeFileCreationTime( - file, - editedTime, - ); - updateExistingFilePubMetadata(file, updatedFile); - } - // eslint-disable-next-line no-constant-condition - if (false) { - const cryptoWorker = - await ComlinkCryptoWorker.getInstance(); - await updateRemotePublicMagicMetadata( - file, - { editedTime }, - cryptoWorker.encryptMetadata, - cryptoWorker.decryptMetadata, - ); - } + const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + await updateRemotePublicMagicMetadata( + file, + { dateTime, dateTimeOffset, editedTime: timestamp }, + cryptoWorker.encryptMetadata, + cryptoWorker.decryptMetadata, + ); log.debug(() => ["after", file.pubMagicMetadata]); diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index ae689797d9..fc6464c874 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -157,6 +157,22 @@ export enum ItemVisibility { * Also see: [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet]. */ export interface PublicMagicMetadata { + /** + * A ISO 8601 date time string without a timezone, indicating the local time + * where the photo was taken. + * + * e.g. "2022-01-26T13:08:20". + * + * See: [Note: Photos are always in local date/time]. + */ + dateTime?: string; + /** + * When available, a "±HH:mm" string indicating the UTC offset for + * {@link dateTime}. + * + * e.g. "+02:00". + */ + dateTimeOffset?: string; /** * Modified value of the date time associated with an {@link EnteFile}. * @@ -575,13 +591,13 @@ export interface ParsedMetadataDate { * This is an optional UTC offset string of the form "±HH:mm" or "Z", * specifying the timezone offset for {@link dateTime} when available. */ - offsetTime: string | undefined; + offset: string | undefined; /** * UTC epoch microseconds derived from {@link dateTime} and - * {@link offsetTime}. + * {@link offset}. * - * When the {@link offsetTime} is present, this will accurately reflect a - * UTC timestamp. When the {@link offsetTime} is not present it convert to a + * When the {@link offset} is present, this will accurately reflect a + * UTC timestamp. When the {@link offset} is not present it convert to a * UTC timestamp by assuming that the given {@link dateTime} is in the local * time where this code is running. This is a good assumption but not always * correct (e.g. vacation photos). @@ -626,7 +642,7 @@ export const parseMetadataDate = ( // Now we try to massage s into two parts - the local date/time string, and // an UTC offset string. - let offsetTime: string | undefined; + let offset: string | undefined; let sWithoutOffset: string; // Check to see if there is a time-zone descriptor of the form "Z" or @@ -634,7 +650,7 @@ export const parseMetadataDate = ( const m = s.match(/Z|[+-]\d\d:?\d\d$/); if (m?.index) { sWithoutOffset = s.substring(0, m.index); - offsetTime = s.substring(m.index); + offset = s.substring(m.index); } else { sWithoutOffset = s; } @@ -674,7 +690,7 @@ export const parseMetadataDate = ( // any time zone descriptor. const dateTime = dropLast(date.toISOString()); - return { dateTime, offsetTime, timestamp }; + return { dateTime, offset, timestamp }; }; const dropLast = (s: string) => (s ? s.substring(0, s.length - 1) : s); diff --git a/web/packages/new/photos/components/PhotoDateTimePicker.tsx b/web/packages/new/photos/components/PhotoDateTimePicker.tsx index cab2a51ddd..86859a3672 100644 --- a/web/packages/new/photos/components/PhotoDateTimePicker.tsx +++ b/web/packages/new/photos/components/PhotoDateTimePicker.tsx @@ -147,14 +147,14 @@ const parseMetadataDateFromDayjs = (d: Dayjs): ParsedMetadataDate => { const s = d.format(); let dateTime: string; - let offsetTime: string | undefined; + let offset: string | undefined; // Check to see if there is a time-zone descriptor of the form "Z" or // "±05:30" or "±0530" at the end of s. const m = s.match(/Z|[+-]\d\d:?\d\d$/); if (m?.index) { dateTime = s.substring(0, m.index); - offsetTime = s.substring(m.index); + offset = s.substring(m.index); } else { throw new Error( `Dayjs.format returned a string "${s}" without a timezone offset`, @@ -163,5 +163,5 @@ const parseMetadataDateFromDayjs = (d: Dayjs): ParsedMetadataDate => { const timestamp = d.valueOf() * 1000; - return { dateTime, offsetTime, timestamp }; + return { dateTime, offset, timestamp }; }; From 657ea68122514d60f539d2c6c354459c21ab05b6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 20:21:56 +0530 Subject: [PATCH 035/211] Keep in sync --- .../FileInfo/RenderCreationTime.tsx | 4 +-- web/packages/media/file-metadata.ts | 33 ++++++++++--------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx index d7b0a21e80..372a1ee3d7 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx @@ -32,7 +32,7 @@ export function RenderCreationTime({ try { setLoading(true); if (isInEditMode && file) { - const { dateTime, dateTimeOffset, timestamp } = pickedTime; + const { dateTime, offset, timestamp } = pickedTime; if (timestamp == file?.metadata.creationTime) { // Same as before. closeEditMode(); @@ -44,7 +44,7 @@ export function RenderCreationTime({ const cryptoWorker = await ComlinkCryptoWorker.getInstance(); await updateRemotePublicMagicMetadata( file, - { dateTime, dateTimeOffset, editedTime: timestamp }, + { dateTime, dateTimeOffset: offset, editedTime: timestamp }, cryptoWorker.encryptMetadata, cryptoWorker.decryptMetadata, ); diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index fc6464c874..07c2dcde84 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -303,11 +303,13 @@ const withoutNullAndUndefinedValues = (o: object) => /** * Update the public magic metadata associated with a file on remote. * - * This function updates the public magic metadata on remote, and also modifies - * the provided {@link EnteFile} object with the updated values in place, but it - * does not update the state of the local databases. The caller needs to ensure - * that we subsequently sync with remote to fetch the updates as part of the - * diff and update the {@link EnteFile} that is persisted in our local db. + * This function updates the public magic metadata on remote, and as a + * convenience also modifies the provided {@link EnteFile} object in place with + * the updated values, but it does not update the state of the local databases. + * + * The caller needs to ensure that we subsequently sync with remote to fetch the + * updates as part of the diff and update the {@link EnteFile} that is persisted + * in our local db. * * @param enteFile The {@link EnteFile} whose public magic metadata we want to * update. @@ -349,17 +351,17 @@ export const updateRemotePublicMagicMetadata = async ( await putFilesPublicMagicMetadata(updateRequest); - // Modify the in-memory object. TODO: This is hacky, and we should find a - // better way, I'm just retaining the existing behaviour. - // - // Also, we need a cast since the underlying pubMagicMetadata type is - // imprecise. + // Modify the in-memory object to use the updated envelope. This steps are + // quite ad-hoc, as is the concept of updating the object in place. enteFile.pubMagicMetadata = updatedEnvelope as typeof enteFile.pubMagicMetadata; - // If the above is hacky, this is even worse. TODO, or at least move to a - // more visible place. + // The correct version will come in the updated EnteFile we get in the + // response of the /diff. Temporarily bump it for the in place edits. enteFile.pubMagicMetadata.version = enteFile.pubMagicMetadata.version + 1; - enteFile.pubMagicMetadata.data = updatedMetadata; + // Re-read the data. + await decryptPublicMagicMetadata(enteFile, decryptMetadataF); + // Re-jig the other bits of EnteFile that depend on its public magic + // metadata. mergeMetadata1(enteFile); }; @@ -425,7 +427,7 @@ interface UpdateMagicMetadataRequest { * metadata JSON object for an {@link enteFile}, using the provided * {@link encryptMetadataF} function to encrypt the JSON. */ -export const updateMagicMetadataRequest = async ( +const updateMagicMetadataRequest = async ( enteFile: EnteFile, metadata: PrivateMagicMetadata | PublicMagicMetadata, metadataVersion: number, @@ -463,6 +465,7 @@ export const updateMagicMetadataRequest = async ( * @param request The list of file ids and the updated encrypted magic metadata * associated with each of them. */ +// TODO: Remove export once this is used. export const putFilesMagicMetadata = async ( request: UpdateMagicMetadataRequest, ) => @@ -480,7 +483,7 @@ export const putFilesMagicMetadata = async ( * @param request The list of file ids and the updated encrypted magic metadata * associated with each of them. */ -export const putFilesPublicMagicMetadata = async ( +const putFilesPublicMagicMetadata = async ( request: UpdateMagicMetadataRequest, ) => ensureOk( From 8e284c1139be14b29d9f134e63c6cf0d1c974596 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 20:28:35 +0530 Subject: [PATCH 036/211] Don't use the offset --- .../PhotoViewer/FileInfo/RenderCreationTime.tsx | 12 ++++++++++-- web/packages/media/file-metadata.ts | 6 +++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx index 372a1ee3d7..885310ee28 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx @@ -32,7 +32,15 @@ export function RenderCreationTime({ try { setLoading(true); if (isInEditMode && file) { - const { dateTime, offset, timestamp } = pickedTime; + // Use the updated date time (both in its canonical dateTime + // form, and also as the legacy timestamp). But don't use the + // offset. The offset here will be the offset of the computer + // where this user is making this edit, not the offset of the + // place where the photo was taken. In a future iteration of the + // date time editor, we can provide functionality for the user + // to edit the associated offset, but right now it is not even + // surfaced, so don't also potentially overwrite it. + const { dateTime, timestamp } = pickedTime; if (timestamp == file?.metadata.creationTime) { // Same as before. closeEditMode(); @@ -44,7 +52,7 @@ export function RenderCreationTime({ const cryptoWorker = await ComlinkCryptoWorker.getInstance(); await updateRemotePublicMagicMetadata( file, - { dateTime, dateTimeOffset: offset, editedTime: timestamp }, + { dateTime, editedTime: timestamp }, cryptoWorker.encryptMetadata, cryptoWorker.decryptMetadata, ); diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 07c2dcde84..0e47280708 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -167,12 +167,12 @@ export interface PublicMagicMetadata { */ dateTime?: string; /** - * When available, a "±HH:mm" string indicating the UTC offset for - * {@link dateTime}. + * When available, a "±HH:mm" string indicating the UTC offset of the place + * where the photo was taken. * * e.g. "+02:00". */ - dateTimeOffset?: string; + offsetTime?: string; /** * Modified value of the date time associated with an {@link EnteFile}. * From 5d16f5735b2408cc0caa1c8fcb90f486643a020a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Tue, 6 Aug 2024 20:39:34 +0530 Subject: [PATCH 037/211] Inline --- .../FileInfo/RenderCreationTime.tsx | 94 ------------------- .../components/PhotoViewer/FileInfo/index.tsx | 90 +++++++++++++++++- 2 files changed, 88 insertions(+), 96 deletions(-) delete mode 100644 web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx deleted file mode 100644 index 885310ee28..0000000000 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/RenderCreationTime.tsx +++ /dev/null @@ -1,94 +0,0 @@ -import log from "@/base/log"; -import { - updateRemotePublicMagicMetadata, - type ParsedMetadataDate, -} from "@/media/file-metadata"; -import { PhotoDateTimePicker } from "@/new/photos/components/PhotoDateTimePicker"; -import { EnteFile } from "@/new/photos/types/file"; -import { FlexWrapper } from "@ente/shared/components/Container"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; -import { formatDate, formatTime } from "@ente/shared/time/format"; -import CalendarTodayIcon from "@mui/icons-material/CalendarToday"; -import { useState } from "react"; -import InfoItem from "./InfoItem"; - -export function RenderCreationTime({ - shouldDisableEdits, - file, - scheduleUpdate, -}: { - shouldDisableEdits: boolean; - file: EnteFile; - scheduleUpdate: () => void; -}) { - const [loading, setLoading] = useState(false); - const originalCreationTime = new Date(file?.metadata.creationTime / 1000); - const [isInEditMode, setIsInEditMode] = useState(false); - - const openEditMode = () => setIsInEditMode(true); - const closeEditMode = () => setIsInEditMode(false); - - const saveEdits = async (pickedTime: ParsedMetadataDate) => { - try { - setLoading(true); - if (isInEditMode && file) { - // Use the updated date time (both in its canonical dateTime - // form, and also as the legacy timestamp). But don't use the - // offset. The offset here will be the offset of the computer - // where this user is making this edit, not the offset of the - // place where the photo was taken. In a future iteration of the - // date time editor, we can provide functionality for the user - // to edit the associated offset, but right now it is not even - // surfaced, so don't also potentially overwrite it. - const { dateTime, timestamp } = pickedTime; - if (timestamp == file?.metadata.creationTime) { - // Same as before. - closeEditMode(); - return; - } - - log.debug(() => ["before", file.pubMagicMetadata]); - - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); - await updateRemotePublicMagicMetadata( - file, - { dateTime, editedTime: timestamp }, - cryptoWorker.encryptMetadata, - cryptoWorker.decryptMetadata, - ); - - log.debug(() => ["after", file.pubMagicMetadata]); - - scheduleUpdate(); - } - } catch (e) { - log.error("failed to update creationTime", e); - } finally { - closeEditMode(); - setLoading(false); - } - }; - - return ( - <> - - } - title={formatDate(originalCreationTime)} - caption={formatTime(originalCreationTime)} - openEditor={openEditMode} - loading={loading} - hideEditOption={shouldDisableEdits || isInEditMode} - /> - {isInEditMode && ( - - )} - - - ); -} diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index 8c1bb6321f..dc32e4baa6 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -1,9 +1,15 @@ import { EnteDrawer } from "@/base/components/EnteDrawer"; import { Titlebar } from "@/base/components/Titlebar"; import { nameAndExtension } from "@/base/file"; +import log from "@/base/log"; import type { ParsedMetadata } from "@/media/file-metadata"; +import { + updateRemotePublicMagicMetadata, + type ParsedMetadataDate, +} from "@/media/file-metadata"; import { FileType } from "@/media/file-type"; import { UnidentifiedFaces } from "@/new/photos/components/PeopleList"; +import { PhotoDateTimePicker } from "@/new/photos/components/PhotoDateTimePicker"; import { photoSwipeZIndex } from "@/new/photos/components/PhotoViewer"; import { tagNumericValue, type RawExifTags } from "@/new/photos/services/exif"; import { isMLEnabled } from "@/new/photos/services/ml"; @@ -12,8 +18,10 @@ import { formattedByteSize } from "@/new/photos/utils/units"; import CopyButton from "@ente/shared/components/CodeBlock/CopyButton"; import { FlexWrapper } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; +import ComlinkCryptoWorker from "@ente/shared/crypto"; import { formatDate, formatTime } from "@ente/shared/time/format"; import BackupOutlined from "@mui/icons-material/BackupOutlined"; +import CalendarTodayIcon from "@mui/icons-material/CalendarToday"; import CameraOutlined from "@mui/icons-material/CameraOutlined"; import FolderOutlined from "@mui/icons-material/FolderOutlined"; import LocationOnOutlined from "@mui/icons-material/LocationOnOutlined"; @@ -44,7 +52,6 @@ import { FileNameEditDialog } from "./FileNameEditDialog"; import InfoItem from "./InfoItem"; import MapBox from "./MapBox"; import { RenderCaption } from "./RenderCaption"; -import { RenderCreationTime } from "./RenderCreationTime"; export interface FileInfoExif { tags: RawExifTags | undefined; @@ -140,7 +147,7 @@ export const FileInfo: React.FC = ({ }} /> - @@ -347,6 +354,85 @@ const FileInfoSidebar = styled((props: DialogProps) => ( }, }); +interface CreationTimeProps { + file: EnteFile; + shouldDisableEdits: boolean; + scheduleUpdate: () => void; +} + +export const CreationTime: React.FC = ({ + file, + shouldDisableEdits, + scheduleUpdate, +}) => { + const [loading, setLoading] = useState(false); + const originalCreationTime = new Date(file?.metadata.creationTime / 1000); + const [isInEditMode, setIsInEditMode] = useState(false); + + const openEditMode = () => setIsInEditMode(true); + const closeEditMode = () => setIsInEditMode(false); + + const saveEdits = async (pickedTime: ParsedMetadataDate) => { + try { + setLoading(true); + if (isInEditMode && file) { + // Use the updated date time (both in its canonical dateTime + // form, and also as the legacy timestamp). But don't use the + // offset. The offset here will be the offset of the computer + // where this user is making this edit, not the offset of the + // place where the photo was taken. In a future iteration of the + // date time editor, we can provide functionality for the user + // to edit the associated offset, but right now it is not even + // surfaced, so don't also potentially overwrite it. + const { dateTime, timestamp } = pickedTime; + if (timestamp == file?.metadata.creationTime) { + // Same as before. + closeEditMode(); + return; + } + + const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + await updateRemotePublicMagicMetadata( + file, + { dateTime, editedTime: timestamp }, + cryptoWorker.encryptMetadata, + cryptoWorker.decryptMetadata, + ); + + scheduleUpdate(); + } + } catch (e) { + log.error("failed to update creationTime", e); + } finally { + closeEditMode(); + setLoading(false); + } + }; + + return ( + <> + + } + title={formatDate(originalCreationTime)} + caption={formatTime(originalCreationTime)} + openEditor={openEditMode} + loading={loading} + hideEditOption={shouldDisableEdits || isInEditMode} + /> + {isInEditMode && ( + + )} + + + ); +}; + interface RenderFileNameProps { file: EnteFile; shouldDisableEdits: boolean; From 36673997dd5cc875a79b13538dba05a56fbe41ee Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 09:37:05 +0530 Subject: [PATCH 038/211] lf --- web/packages/media/file-metadata.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 0e47280708..0c12358a12 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -288,10 +288,16 @@ export const decryptPublicMagicMetadata = async ( withoutNullAndUndefinedValues(jsonValue as object), ); - // @ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] + // -@ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] + // We can't use -@ts-expect-error since this code is also included in the + // packages which don't have strict mode enabled (and thus don't error). + // eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error + // @ts-ignore envelope.data = result; - // @ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] + // -@ts-expect-error [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] + // eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error + // @ts-ignore return result; }; From 71b909d9500842a0f814fb8c9537b45fd6149cd2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 10:15:00 +0530 Subject: [PATCH 039/211] Outline --- web/packages/media/file-metadata.ts | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 0c12358a12..05896c0495 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -703,3 +703,40 @@ export const parseMetadataDate = ( }; const dropLast = (s: string) => (s ? s.substring(0, s.length - 1) : s); + +/** + * Return a date that can be used on the UI from a {@link ParsedMetadataDate}, + * or its {@link dateTime} component, or the legacy epoch timestamps. + * + * These dates are all hypothetically in the timezone of the place where the + * photo was taken. Different photos might've been taken in different timezones, + * which is why it is hypothetical, so concretely these are all mapped to the + * current timezone. + * + * The difference is subtle, but we should not think of these as absolute points + * on the UTC timeline. They are instead better thought of as dates without an + * associated timezone. For the purpose of mapping them all to a comparable + * dimension them we all contingently use the current timezone - this makes it + * easy to use JavaScript Date constructor which assumes that any date/time + * string without an associated timezone is in the current timezone. + * + * Whenever we're surfacing them in the UI, or using them for grouping (say by + * day), we should use their current timezone representation, not the UTC one. + * + * See also: [Note: Photos are always in local date/time]. + */ +export const toUIDate = (dateLike: ParsedMetadataDate | string | number) => { + switch (typeof dateLike) { + case "object": + // A ISO 8601 string without a timezone. The Date constructor will + // assume the timezone to be the current timezone. + return new Date(dateLike.dateTime); + case "string": + // This is expected to be a string with the same meaning as + // `ParsedMetadataDate.dateTime`. + return new Date(dateLike); + case "number": + // A legacy epoch microseconds value. + return new Date(dateLike / 1000); + } +}; From 7ecfa20f03d850079ccaae0a63d645daa6824af1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 10:19:12 +0530 Subject: [PATCH 040/211] Up --- web/packages/media/file-metadata.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 05896c0495..acd01c86d8 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -306,6 +306,21 @@ const withoutNullAndUndefinedValues = (o: object) => Object.entries(o).filter(([, v]) => v !== null && v !== undefined), ); +/** + * Return the file's creation date in a form suitable for using in the UI. + * + * For all the details and nuance, see {@link toUIDate}. + */ +export const getUICreationDate = ( + enteFile: EnteFile, + publicMagicMetadata: PublicMagicMetadata, +) => + toUIDate( + publicMagicMetadata.dateTime ?? + publicMagicMetadata.editedTime ?? + enteFile.metadata.creationTime, + ); + /** * Update the public magic metadata associated with a file on remote. * From f4adea5a6052c5482c004b38abf451c0c89b37b3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 10:33:53 +0530 Subject: [PATCH 041/211] Add wrapper --- web/packages/shared/file-metadata.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 web/packages/shared/file-metadata.ts diff --git a/web/packages/shared/file-metadata.ts b/web/packages/shared/file-metadata.ts new file mode 100644 index 0000000000..31bffc143e --- /dev/null +++ b/web/packages/shared/file-metadata.ts @@ -0,0 +1,16 @@ +import { decryptPublicMagicMetadata } from "@/media/file-metadata"; +import { EnteFile } from "@/new/photos/types/file"; +import ComlinkCryptoWorker from "@ente/shared/crypto"; + +/** + * On-demand decrypt the public magic metadata for an {@link EnteFile} for code + * running on the main thread. + * + * It both modifies the given file object, and also returns the decrypted + * metadata. + */ +export const getPublicMagicMetadataMT = async (enteFile: EnteFile) => + decryptPublicMagicMetadata( + enteFile, + (await ComlinkCryptoWorker.getInstance()).decryptMetadata, + ); From 62436db1c308f05516bc32d827044b14981a3750 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 10:56:25 +0530 Subject: [PATCH 042/211] Add sync variant --- web/packages/shared/file-metadata.ts | 34 +++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/web/packages/shared/file-metadata.ts b/web/packages/shared/file-metadata.ts index 31bffc143e..8ac145d057 100644 --- a/web/packages/shared/file-metadata.ts +++ b/web/packages/shared/file-metadata.ts @@ -1,6 +1,12 @@ -import { decryptPublicMagicMetadata } from "@/media/file-metadata"; +import { + decryptPublicMagicMetadata, + type PublicMagicMetadata, +} from "@/media/file-metadata"; import { EnteFile } from "@/new/photos/types/file"; import ComlinkCryptoWorker from "@ente/shared/crypto"; +import { decryptMetadata } from "packages/base/crypto/ente"; +import { isDevBuild } from "packages/base/env"; +import { fileLogID } from "packages/new/photos/utils/file"; /** * On-demand decrypt the public magic metadata for an {@link EnteFile} for code @@ -14,3 +20,29 @@ export const getPublicMagicMetadataMT = async (enteFile: EnteFile) => enteFile, (await ComlinkCryptoWorker.getInstance()).decryptMetadata, ); + +/** + * On-demand decrypt the public magic metadata for an {@link EnteFile} for code + * running on the main thread, but do it synchronously. + * + * It both modifies the given file object, and also returns the decrypted + * metadata. + * + * We are not expected to be in a scenario where the file gets to the UI without + * having its public magic metadata decrypted, so this function is a sanity + * check and should be a no-op in usually. On debug builds it'll throw if it + * finds its assumptions broken. + */ +export const getPublicMagicMetadataMTSync = (enteFile: EnteFile) => { + if (!enteFile.pubMagicMetadata) return undefined; + if (typeof enteFile.pubMagicMetadata.data == "string") { + if (isDevBuild) + throw new Error( + `Public magic metadata for ${fileLogID(enteFile)} had not been decrypted even when the file reached the UI layer`, + ); + decryptPublicMagicMetadata(enteFile, decryptMetadata); + } + // This cast is unavoidable in the current setup. We need to refactor the + // types so that this cast in not needed. + return enteFile.pubMagicMetadata.data as PublicMagicMetadata; +}; From 74e50a8e376441f3023f4e3151b8ff79bf5d4c20 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 11:01:06 +0530 Subject: [PATCH 043/211] Show from both places --- .../components/PhotoViewer/FileInfo/index.tsx | 24 +++++++++++-------- web/packages/shared/file-metadata.ts | 6 ++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index dc32e4baa6..c947c577f2 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -4,6 +4,7 @@ import { nameAndExtension } from "@/base/file"; import log from "@/base/log"; import type { ParsedMetadata } from "@/media/file-metadata"; import { + getUICreationDate, updateRemotePublicMagicMetadata, type ParsedMetadataDate, } from "@/media/file-metadata"; @@ -19,6 +20,7 @@ import CopyButton from "@ente/shared/components/CodeBlock/CopyButton"; import { FlexWrapper } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; import ComlinkCryptoWorker from "@ente/shared/crypto"; +import { getPublicMagicMetadataMTSync } from "@ente/shared/file-metadata"; import { formatDate, formatTime } from "@ente/shared/time/format"; import BackupOutlined from "@mui/icons-material/BackupOutlined"; import CalendarTodayIcon from "@mui/icons-material/CalendarToday"; @@ -148,7 +150,7 @@ export const FileInfo: React.FC = ({ /> ( }); interface CreationTimeProps { - file: EnteFile; + enteFile: EnteFile; shouldDisableEdits: boolean; scheduleUpdate: () => void; } export const CreationTime: React.FC = ({ - file, + enteFile, shouldDisableEdits, scheduleUpdate, }) => { const [loading, setLoading] = useState(false); - const originalCreationTime = new Date(file?.metadata.creationTime / 1000); const [isInEditMode, setIsInEditMode] = useState(false); const openEditMode = () => setIsInEditMode(true); const closeEditMode = () => setIsInEditMode(false); + const publicMagicMetadata = getPublicMagicMetadataMTSync(enteFile); + const originalDate = getUICreationDate(enteFile, publicMagicMetadata); + const saveEdits = async (pickedTime: ParsedMetadataDate) => { try { setLoading(true); - if (isInEditMode && file) { + if (isInEditMode && enteFile) { // Use the updated date time (both in its canonical dateTime // form, and also as the legacy timestamp). But don't use the // offset. The offset here will be the offset of the computer @@ -385,7 +389,7 @@ export const CreationTime: React.FC = ({ // to edit the associated offset, but right now it is not even // surfaced, so don't also potentially overwrite it. const { dateTime, timestamp } = pickedTime; - if (timestamp == file?.metadata.creationTime) { + if (timestamp == originalDate.getTime()) { // Same as before. closeEditMode(); return; @@ -393,7 +397,7 @@ export const CreationTime: React.FC = ({ const cryptoWorker = await ComlinkCryptoWorker.getInstance(); await updateRemotePublicMagicMetadata( - file, + enteFile, { dateTime, editedTime: timestamp }, cryptoWorker.encryptMetadata, cryptoWorker.decryptMetadata, @@ -414,15 +418,15 @@ export const CreationTime: React.FC = ({ } - title={formatDate(originalCreationTime)} - caption={formatTime(originalCreationTime)} + title={formatDate(originalDate)} + caption={formatTime(originalDate)} openEditor={openEditMode} loading={loading} hideEditOption={shouldDisableEdits || isInEditMode} /> {isInEditMode && ( Date: Wed, 7 Aug 2024 11:41:29 +0530 Subject: [PATCH 044/211] Use in fixer --- .../photos/src/components/FixCreationTime.tsx | 63 +++++++++++++------ .../components/PhotoViewer/FileInfo/index.tsx | 2 + 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/web/apps/photos/src/components/FixCreationTime.tsx b/web/apps/photos/src/components/FixCreationTime.tsx index 836da7924a..9018a48022 100644 --- a/web/apps/photos/src/components/FixCreationTime.tsx +++ b/web/apps/photos/src/components/FixCreationTime.tsx @@ -1,12 +1,19 @@ import log from "@/base/log"; -import type { ParsedMetadataDate } from "@/media/file-metadata"; +import { + decryptPublicMagicMetadata, + getUICreationDate, + updateRemotePublicMagicMetadata, + type ParsedMetadataDate, +} from "@/media/file-metadata"; import { FileType } from "@/media/file-type"; import { PhotoDateTimePicker } from "@/new/photos/components/PhotoDateTimePicker"; import downloadManager from "@/new/photos/services/download"; import { extractExifDates } from "@/new/photos/services/exif"; import { EnteFile } from "@/new/photos/types/file"; import { fileLogID } from "@/new/photos/utils/file"; +import { ensure } from "@/utils/ensure"; import DialogBox from "@ente/shared/components/DialogBox/"; +import ComlinkCryptoWorker from "@ente/shared/crypto"; import { Button, FormControl, @@ -21,10 +28,6 @@ import { useFormik } from "formik"; import { t } from "i18next"; import { GalleryContext } from "pages/gallery"; import React, { useContext, useEffect, useState } from "react"; -import { - changeFileCreationTime, - updateExistingFilePubMetadata, -} from "utils/file"; /** The current state of the fixing process. */ type Status = "running" | "completed" | "completed-with-errors"; @@ -277,7 +280,7 @@ type SetProgressTracker = React.Dispatch< const updateFiles = async ( enteFiles: EnteFile[], fixOption: FixOption, - customDate: ParsedMetadataDate, + customDate: ParsedMetadataDate | undefined, setProgressTracker: SetProgressTracker, ) => { setProgressTracker({ current: 0, total: enteFiles.length }); @@ -312,25 +315,29 @@ const updateFiles = async ( * {@link fixOption} is provided, but the given underlying image for the given * {@link enteFile} does not have a corresponding Exif (or related) value, then * that file is skipped. - * - * Note that metadata associated with an {@link EnteFile} is immutable, and we - * instead modify the mutable metadata section associated with the file. See - * [Note: Metadatum] for more details. */ -export const updateEnteFileDate = async ( +const updateEnteFileDate = async ( enteFile: EnteFile, fixOption: FixOption, - customDate: ParsedMetadataDate, + customDate: ParsedMetadataDate | undefined, ) => { let newDate: ParsedMetadataDate | undefined; - if (fixOption === "custom") { - newDate = customDate; + + if (fixOption == "custom") { + newDate = { + dateTime: ensure(customDate).dateTime, + // See [Note: Don't modify offsetTime when editing date via picker] + // for why we don't also set the offset here. + offset: undefined, + timestamp: ensure(customDate).timestamp, + }; } else if (enteFile.metadata.fileType == FileType.image) { const stream = await downloadManager.getFile(enteFile); const blob = await new Response(stream).blob(); const file = new File([blob], enteFile.metadata.title); const { DateTimeOriginal, DateTimeDigitized, MetadataDate, DateTime } = await extractExifDates(file); + switch (fixOption) { case "date-time-original": newDate = DateTimeOriginal ?? DateTime; @@ -344,11 +351,27 @@ export const updateEnteFileDate = async ( } } - if (newDate && newDate.timestamp !== enteFile.metadata.creationTime) { - const updatedFile = await changeFileCreationTime( + if (!newDate) return; + + const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + + const existingUIDate = getUICreationDate( + enteFile, + await decryptPublicMagicMetadata( enteFile, - newDate.timestamp, - ); - updateExistingFilePubMetadata(enteFile, updatedFile); - } + cryptoWorker.decryptMetadata, + ), + ); + if (newDate.timestamp == existingUIDate.getTime()) return; + + await updateRemotePublicMagicMetadata( + enteFile, + { + dateTime: newDate.dateTime, + offsetTime: newDate.offset, + editedTime: newDate.timestamp, + }, + cryptoWorker.encryptMetadata, + cryptoWorker.decryptMetadata, + ); }; diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index c947c577f2..25ce1d61b1 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -380,6 +380,8 @@ export const CreationTime: React.FC = ({ try { setLoading(true); if (isInEditMode && enteFile) { + // [Note: Don't modify offsetTime when editing date via picker] + // // Use the updated date time (both in its canonical dateTime // form, and also as the legacy timestamp). But don't use the // offset. The offset here will be the offset of the computer From 9cc8469ed9017f016420488e2042a228ed64772e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 12:04:34 +0530 Subject: [PATCH 045/211] Remove unused --- web/apps/photos/src/utils/file/index.ts | 19 ------------------- web/packages/media/file-metadata.ts | 12 +++++++----- 2 files changed, 7 insertions(+), 24 deletions(-) diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 71978ee24c..6857585aa9 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -210,25 +210,6 @@ export async function changeFilesVisibility( return await updateFileMagicMetadata(fileWithUpdatedMagicMetadataList); } -export async function changeFileCreationTime( - file: EnteFile, - editedTime: number, -): Promise { - const updatedPublicMagicMetadataProps: FilePublicMagicMetadataProps = { - editedTime, - }; - const updatedPublicMagicMetadata: FilePublicMagicMetadata = - await updateMagicMetadata( - updatedPublicMagicMetadataProps, - file.pubMagicMetadata, - file.key, - ); - const updateResult = await updateFilePublicMagicMetadata([ - { file, updatedPublicMagicMetadata }, - ]); - return updateResult[0]; -} - export async function changeFileName( file: EnteFile, editedName: string, diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index acd01c86d8..02591a0374 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -220,12 +220,14 @@ const PublicMagicMetadata = z .object({ // [Note: Zod doesn't work with `exactOptionalPropertyTypes` yet] // - // Using `optional` is accurate here. The key is optional, but the value - // itself is not optional. Zod doesn't work with - // `exactOptionalPropertyTypes` yet, but it seems to be on the roadmap so we - // suppress these mismatches. + // Using `optional` is not accurate here. The key is optional, but the + // value itself is not optional. // - // See: https://github.com/colinhacks/zod/issues/635#issuecomment-2196579063 + // Zod doesn't work with `exactOptionalPropertyTypes` yet, but it seems + // to be on the roadmap so we suppress these mismatches. + // + // See: + // https://github.com/colinhacks/zod/issues/635#issuecomment-2196579063 editedTime: z.number().optional(), }) .passthrough(); From 25c97dea48cd81ddbd87af08b4040686ca0af5ec Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 12:21:17 +0530 Subject: [PATCH 046/211] Switch --- .../src/services/upload/uploadService.ts | 19 ++++++------------- web/packages/new/photos/services/exif.ts | 5 ----- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 8511136352..5eeb7ddbad 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -3,10 +3,10 @@ import { basename } from "@/base/file"; import log from "@/base/log"; import { CustomErrorMessage } from "@/base/types/ipc"; import { hasFileHash } from "@/media/file"; -import type { Metadata } from "@/media/file-metadata"; +import type { Metadata, ParsedMetadata } from "@/media/file-metadata"; import { FileType, type FileTypeInfo } from "@/media/file-type"; import { encodeLivePhoto } from "@/media/live-photo"; -import { cmpNewLib, extractExif, wipNewLib } from "@/new/photos/services/exif"; +import { extractExif } from "@/new/photos/services/exif"; import * as ffmpeg from "@/new/photos/services/ffmpeg"; import type { UploadItem } from "@/new/photos/services/upload/types"; import { @@ -31,7 +31,6 @@ import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worke import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/internal/libsodium"; import { CustomError, handleUploadError } from "@ente/shared/error"; -import { parseImageMetadata } from "@ente/shared/utils/exif-old"; import type { Remote } from "comlink"; import { addToCollection } from "services/collectionService"; import { @@ -807,11 +806,10 @@ const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { height: null, }; -async function tryExtractImageMetadata( +const tryExtractImageMetadata = async ( uploadItem: UploadItem, - fileTypeInfo: FileTypeInfo, lastModifiedMs: number, -): Promise { +): Promise => { let file: File; if (typeof uploadItem == "string" || Array.isArray(uploadItem)) { // The library we use for extracting Exif from images, ExifReader, @@ -829,17 +827,12 @@ async function tryExtractImageMetadata( } try { - const oldLib = await parseImageMetadata(file, fileTypeInfo); - if (await wipNewLib()) { - const newLib = await extractExif(file); - cmpNewLib(oldLib, newLib); - } - return oldLib; + return extractExif(file); } catch (e) { log.error(`Failed to extract image metadata for ${uploadItem}`, e); return undefined; } -} +}; const tryExtractVideoMetadata = async (uploadItem: UploadItem) => { try { diff --git a/web/packages/new/photos/services/exif.ts b/web/packages/new/photos/services/exif.ts index 28da2882a0..d559b9750a 100644 --- a/web/packages/new/photos/services/exif.ts +++ b/web/packages/new/photos/services/exif.ts @@ -1,4 +1,3 @@ -import { isDevBuild } from "@/base/env"; import { nameAndExtension } from "@/base/file"; import log from "@/base/log"; import { @@ -11,10 +10,6 @@ import { parseImageMetadata } from "@ente/shared/utils/exif-old"; import ExifReader from "exifreader"; import type { EnteFile } from "../types/file"; import type { ParsedExtractedMetadata } from "../types/metadata"; -import { isInternalUser } from "./feature-flags"; - -// TODO: Exif: WIP flag to inspect the migration from old to new lib. -export const wipNewLib = async () => isDevBuild && (await isInternalUser()); const cmpTsEq = (a: number | undefined | null, b: number | undefined) => { if (!a && !b) return true; From 05725dfdeb8f31161955e65957f90db9c659ed28 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 12:26:02 +0530 Subject: [PATCH 047/211] lay of the land --- .../new/photos/services/ffmpeg/index.ts | 55 +++++++++++-------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/web/packages/new/photos/services/ffmpeg/index.ts b/web/packages/new/photos/services/ffmpeg/index.ts index 71612f2314..c01bf0616a 100644 --- a/web/packages/new/photos/services/ffmpeg/index.ts +++ b/web/packages/new/photos/services/ffmpeg/index.ts @@ -1,6 +1,7 @@ import { ensureElectron } from "@/base/electron"; import type { Electron } from "@/base/types/ipc"; import { ComlinkWorker } from "@/base/worker/comlink-worker"; +import type { ParsedMetadata } from "@/media/file-metadata"; import { NULL_LOCATION, toDataOrPathOrZipEntry, @@ -97,7 +98,7 @@ const makeGenThumbnailCommand = (seekTime: number) => [ * * When we're running in the context of our desktop app _and_ we're passed a * file path , this uses the native FFmpeg bundled with our desktop app. - * Otherwise it uses a wasm FFmpeg running in a web worker. + * Otherwise it uses a wasm build of FFmpeg running in a web worker. * * This function is called during upload, when we need to extract the metadata * of videos that the user is uploading. @@ -108,34 +109,36 @@ const makeGenThumbnailCommand = (seekTime: number) => [ */ export const extractVideoMetadata = async ( uploadItem: UploadItem, -): Promise => { +): Promise => { const command = extractVideoMetadataCommand; - const outputData = + return parseFFmpegExtractedMetadata( uploadItem instanceof File ? await ffmpegExecWeb(command, uploadItem, "txt") : await ensureElectron().ffmpegExec( command, toDataOrPathOrZipEntry(uploadItem), "txt", - ); - - return parseFFmpegExtractedMetadata(outputData); + ), + ); }; -// Options: -// -// - `-c [short for codex] copy` -// - copy is the [stream_specifier](ffmpeg.org/ffmpeg.html#Stream-specifiers) -// - copies all the stream without re-encoding -// -// - `-map_metadata` -// - http://ffmpeg.org/ffmpeg.html#Advanced-options (search for map_metadata) -// - copies all stream metadata to the output -// -// - `-f ffmetadata` -// - https://ffmpeg.org/ffmpeg-formats.html#Metadata-1 -// - dump metadata from media files into a simple INI-like utf-8 text file -// +/** + * The FFmpeg command to use to extract metadata from videos. + * + * Options: + * + * - `-c [short for codex] copy` + * - copy is the [stream_specifier](ffmpeg.org/ffmpeg.html#Stream-specifiers) + * - copies all the stream without re-encoding + * + * - `-map_metadata` + * - http://ffmpeg.org/ffmpeg.html#Advanced-options (search for map_metadata) + * - copies all stream metadata to the output + * + * - `-f ffmetadata` + * - https://ffmpeg.org/ffmpeg-formats.html#Metadata-1 + * - dump metadata from media files into a simple INI-like utf-8 text file + */ const extractVideoMetadataCommand = [ ffmpegPathPlaceholder, "-i", @@ -158,8 +161,14 @@ enum MetadataTags { LOCATION = "location", } -function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { - const metadataString = new TextDecoder().decode(encodedMetadata); +/** + * Convert the output produced by running the FFmpeg + * {@link extractVideoMetadataCommand} into a {@link ParsedMetadata}. + * + * @param ffmpegOutput The bytes containing the output of the FFmpeg command. + */ +const parseFFmpegExtractedMetadata = (ffmpegOutput: Uint8Array) => { + const metadataString = new TextDecoder().decode(ffmpegOutput); const metadataPropertyArray = metadataString.split("\n"); const metadataKeyValueArray = metadataPropertyArray.map((property) => property.split("="), @@ -189,7 +198,7 @@ function parseFFmpegExtractedMetadata(encodedMetadata: Uint8Array) { height: null, }; return parsedMetadata; -} +}; const parseAppleISOLocation = (isoLocation: string | undefined) => { let location = { ...NULL_LOCATION }; From 1fc1d3f4c028a481d1c32f47cdda24b969c8d654 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 12:30:49 +0530 Subject: [PATCH 048/211] Rearrange --- .../new/photos/services/ffmpeg/index.ts | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/web/packages/new/photos/services/ffmpeg/index.ts b/web/packages/new/photos/services/ffmpeg/index.ts index c01bf0616a..1a20e2ce65 100644 --- a/web/packages/new/photos/services/ffmpeg/index.ts +++ b/web/packages/new/photos/services/ffmpeg/index.ts @@ -136,7 +136,7 @@ export const extractVideoMetadata = async ( * - copies all stream metadata to the output * * - `-f ffmetadata` - * - https://ffmpeg.org/ffmpeg-formats.html#Metadata-1 + * - https://ffmpeg.org/ffmpeg-formats.html#Metadata-2 * - dump metadata from media files into a simple INI-like utf-8 text file */ const extractVideoMetadataCommand = [ @@ -168,25 +168,25 @@ enum MetadataTags { * @param ffmpegOutput The bytes containing the output of the FFmpeg command. */ const parseFFmpegExtractedMetadata = (ffmpegOutput: Uint8Array) => { - const metadataString = new TextDecoder().decode(ffmpegOutput); - const metadataPropertyArray = metadataString.split("\n"); - const metadataKeyValueArray = metadataPropertyArray.map((property) => - property.split("="), - ); - const validKeyValuePairs = metadataKeyValueArray.filter( - (keyValueArray) => keyValueArray.length == 2, - ) as [string, string][]; + // The output is a utf8 INI-like text file with key=value pairs interspersed + // with comments and newlines. + // + // https://ffmpeg.org/ffmpeg-formats.html#Metadata-2 + + const lines = new TextDecoder().decode(ffmpegOutput).split("\n"); + const isPair = (xs: string[]): xs is [string, string] => xs.length == 2; + const kvPairs = lines.map((property) => property.split("=")).filter(isPair); - const metadataMap = Object.fromEntries(validKeyValuePairs); + const kv = new Map(kvPairs); const location = parseAppleISOLocation( - metadataMap[MetadataTags.APPLE_LOCATION_ISO] ?? - metadataMap[MetadataTags.LOCATION], + kv.get(MetadataTags.APPLE_LOCATION_ISO) ?? + kv.get(MetadataTags.LOCATION), ); const creationTime = parseCreationTime( - metadataMap[MetadataTags.APPLE_CREATION_DATE] ?? - metadataMap[MetadataTags.CREATION_TIME], + kv.get(MetadataTags.APPLE_CREATION_DATE) ?? + kv.get(MetadataTags.CREATION_TIME), ); const parsedMetadata: ParsedExtractedMetadata = { creationTime, From 6adbb82d54bca0f0ac35ed2506a704a3eb11a038 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 12:39:32 +0530 Subject: [PATCH 049/211] p1 --- .../new/photos/services/ffmpeg/index.ts | 65 +++++++++++-------- 1 file changed, 38 insertions(+), 27 deletions(-) diff --git a/web/packages/new/photos/services/ffmpeg/index.ts b/web/packages/new/photos/services/ffmpeg/index.ts index 1a20e2ce65..3ce88677a3 100644 --- a/web/packages/new/photos/services/ffmpeg/index.ts +++ b/web/packages/new/photos/services/ffmpeg/index.ts @@ -1,14 +1,13 @@ import { ensureElectron } from "@/base/electron"; +import log from "@/base/log"; import type { Electron } from "@/base/types/ipc"; import { ComlinkWorker } from "@/base/worker/comlink-worker"; import type { ParsedMetadata } from "@/media/file-metadata"; import { - NULL_LOCATION, toDataOrPathOrZipEntry, type DesktopUploadItem, type UploadItem, } from "@/new/photos/services/upload/types"; -import type { ParsedExtractedMetadata } from "@/new/photos/types/metadata"; import { readConvertToMP4Done, readConvertToMP4Stream, @@ -179,37 +178,49 @@ const parseFFmpegExtractedMetadata = (ffmpegOutput: Uint8Array) => { const kv = new Map(kvPairs); - const location = parseAppleISOLocation( - kv.get(MetadataTags.APPLE_LOCATION_ISO) ?? - kv.get(MetadataTags.LOCATION), + const location = parseMetadataLocation( + kv.get("com.apple.quicktime.location.ISO6709") ?? kv.get("location"), ); - const creationTime = parseCreationTime( - kv.get(MetadataTags.APPLE_CREATION_DATE) ?? - kv.get(MetadataTags.CREATION_TIME), - ); - const parsedMetadata: ParsedExtractedMetadata = { - creationTime, - location: { - latitude: location.latitude, - longitude: location.longitude, - }, - width: null, - height: null, - }; - return parsedMetadata; + return { location }; + // const creationTime = parseCreationTime( + // kv.get(MetadataTags.APPLE_CREATION_DATE) ?? + // kv.get(MetadataTags.CREATION_TIME), + // ); + // const parsedMetadata: ParsedExtractedMetadata = { + // creationTime, + // location: { + // latitude: location.latitude, + // longitude: location.longitude, + // }, + // width: null, + // height: null, + // }; + // return parsedMetadata; }; -const parseAppleISOLocation = (isoLocation: string | undefined) => { - let location = { ...NULL_LOCATION }; - if (isoLocation) { - const m = isoLocation - .match(/(\+|-)\d+\.*\d+/g) - ?.map((x) => parseFloat(x)); +/** + * Parse a location string found in the FFmpeg metadata attributes. + * + * This is meant to parse either the "com.apple.quicktime.location.ISO6709" + * (preferable) or the "location" key (fallback). + */ +const parseMetadataLocation = (s: string | undefined) => { + if (!s) return undefined; - location = { latitude: m?.at(0) ?? null, longitude: m?.at(1) ?? null }; + const m = s.match(/(\+|-)\d+\.*\d+/g); + if (!m) { + log.warn(`Ignoring unparseable location string "${s}"`); + return undefined; } - return location; + + const [latitude, longitude] = m.map(parseFloat); + if (!latitude || !longitude) { + log.warn(`Ignoring unparseable location string "${s}"`); + return undefined; + } + + return { latitude, longitude }; }; const parseCreationTime = (creationTime: string | undefined) => { From 728c3a80f461926d5b6a96d6de29fd345dd5c4ad Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 12:47:15 +0530 Subject: [PATCH 050/211] Scaffold --- .../new/photos/services/ffmpeg/index.ts | 51 +++++++++++++------ 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/web/packages/new/photos/services/ffmpeg/index.ts b/web/packages/new/photos/services/ffmpeg/index.ts index 3ce88677a3..134ee86797 100644 --- a/web/packages/new/photos/services/ffmpeg/index.ts +++ b/web/packages/new/photos/services/ffmpeg/index.ts @@ -178,25 +178,19 @@ const parseFFmpegExtractedMetadata = (ffmpegOutput: Uint8Array) => { const kv = new Map(kvPairs); + const result: ParsedMetadata = {}; + const location = parseMetadataLocation( kv.get("com.apple.quicktime.location.ISO6709") ?? kv.get("location"), ); + if (location) result.location = location; + + const creationDate = parseMetadataCreationDate( + kv.get("com.apple.quicktime.creationdate") ?? kv.get("creation_time"), + ); + if (creationDate) result.creationDate = creationDate; - return { location }; - // const creationTime = parseCreationTime( - // kv.get(MetadataTags.APPLE_CREATION_DATE) ?? - // kv.get(MetadataTags.CREATION_TIME), - // ); - // const parsedMetadata: ParsedExtractedMetadata = { - // creationTime, - // location: { - // latitude: location.latitude, - // longitude: location.longitude, - // }, - // width: null, - // height: null, - // }; - // return parsedMetadata; + return result; }; /** @@ -223,6 +217,33 @@ const parseMetadataLocation = (s: string | undefined) => { return { latitude, longitude }; }; +/** + * Parse a date/time string found in the FFmpeg metadata attributes. + * + * This is meant to parse either the "com.apple.quicktime.creationdate" + * (preferable) or the "creation_time" key (fallback). + * + * Both of them are expected to be ISO 8601 date/time strings, but in particular + * the quicktime.creationdate includes the time zone offset. + */ +const parseMetadataCreationDate = (s: string | undefined) => { + if (!s) return undefined; + + const m = s.match(/(\+|-)\d+\.*\d+/g); + if (!m) { + log.warn(`Ignoring unparseable location string "${s}"`); + return undefined; + } + + const [latitude, longitude] = m.map(parseFloat); + if (!latitude || !longitude) { + log.warn(`Ignoring unparseable location string "${s}"`); + return undefined; + } + + return { latitude, longitude }; +}; + const parseCreationTime = (creationTime: string | undefined) => { let dateTime = null; if (creationTime) { From 075096258f99b00d7a4fae959c0f3ef6e9fec43f Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 7 Aug 2024 12:56:39 +0530 Subject: [PATCH 051/211] Rename --- server/ente/filedata/putfiledata.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go index 6a5c5b0630..dcdab16c19 100644 --- a/server/ente/filedata/putfiledata.go +++ b/server/ente/filedata/putfiledata.go @@ -13,8 +13,8 @@ type PutFileDataRequest struct { // ObjectKey is the key of the object in the S3 bucket. This is needed while putting the object in the S3 bucket. ObjectKey *string `json:"objectKey,omitempty"` // size of the object that is being uploaded. This helps in checking the size of the object that is being uploaded. - Size *int64 `json:"size,omitempty"` - Version *int `json:"version,omitempty"` + ObjectSize *int64 `json:"objectSize,omitempty"` + Version *int `json:"version,omitempty"` } func (r PutFileDataRequest) Validate() error { @@ -24,11 +24,11 @@ func (r PutFileDataRequest) Validate() error { // the video playlist is uploaded as part of encrypted data and decryption header return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for preview video") } - if r.Size == nil || r.ObjectKey == nil { + if r.ObjectSize == nil || r.ObjectKey == nil { return ente.NewBadRequestWithMessage("size and objectKey are required for preview video") } case ente.PreviewImage: - if r.Size == nil || r.ObjectKey == nil { + if r.ObjectSize == nil || r.ObjectKey == nil { return ente.NewBadRequestWithMessage("size and objectKey are required for preview image") } case ente.DerivedMeta: From 5c0a80415d40cc3502e8488c83a56ed4a3d44f1b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 12:56:59 +0530 Subject: [PATCH 052/211] Give both a shot --- .../new/photos/services/ffmpeg/index.ts | 51 ++++++++----------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/web/packages/new/photos/services/ffmpeg/index.ts b/web/packages/new/photos/services/ffmpeg/index.ts index 134ee86797..776764fc6f 100644 --- a/web/packages/new/photos/services/ffmpeg/index.ts +++ b/web/packages/new/photos/services/ffmpeg/index.ts @@ -2,7 +2,7 @@ import { ensureElectron } from "@/base/electron"; import log from "@/base/log"; import type { Electron } from "@/base/types/ipc"; import { ComlinkWorker } from "@/base/worker/comlink-worker"; -import type { ParsedMetadata } from "@/media/file-metadata"; +import { parseMetadataDate, type ParsedMetadata } from "@/media/file-metadata"; import { toDataOrPathOrZipEntry, type DesktopUploadItem, @@ -13,7 +13,6 @@ import { readConvertToMP4Stream, writeConvertToMP4Stream, } from "@/new/photos/utils/native-stream"; -import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; import type { Remote } from "comlink"; import { ffmpegPathPlaceholder, @@ -180,14 +179,15 @@ const parseFFmpegExtractedMetadata = (ffmpegOutput: Uint8Array) => { const result: ParsedMetadata = {}; - const location = parseMetadataLocation( - kv.get("com.apple.quicktime.location.ISO6709") ?? kv.get("location"), - ); + const location = + parseFFMetadataLocation( + kv.get("com.apple.quicktime.location.ISO6709"), + ) ?? parseFFMetadataLocation(kv.get("location")); if (location) result.location = location; - const creationDate = parseMetadataCreationDate( - kv.get("com.apple.quicktime.creationdate") ?? kv.get("creation_time"), - ); + const creationDate = + parseFFMetadataDate(kv.get("com.apple.quicktime.creationdate")) ?? + parseFFMetadataDate(kv.get("creation_time")); if (creationDate) result.creationDate = creationDate; return result; @@ -199,7 +199,7 @@ const parseFFmpegExtractedMetadata = (ffmpegOutput: Uint8Array) => { * This is meant to parse either the "com.apple.quicktime.location.ISO6709" * (preferable) or the "location" key (fallback). */ -const parseMetadataLocation = (s: string | undefined) => { +const parseFFMetadataLocation = (s: string | undefined) => { if (!s) return undefined; const m = s.match(/(\+|-)\d+\.*\d+/g); @@ -210,7 +210,7 @@ const parseMetadataLocation = (s: string | undefined) => { const [latitude, longitude] = m.map(parseFloat); if (!latitude || !longitude) { - log.warn(`Ignoring unparseable location string "${s}"`); + log.warn(`Ignoring unparseable video metadata location string "${s}"`); return undefined; } @@ -223,35 +223,26 @@ const parseMetadataLocation = (s: string | undefined) => { * This is meant to parse either the "com.apple.quicktime.creationdate" * (preferable) or the "creation_time" key (fallback). * - * Both of them are expected to be ISO 8601 date/time strings, but in particular - * the quicktime.creationdate includes the time zone offset. + * Both of these are expected to be ISO 8601 date/time strings, but we prefer + * "com.apple.quicktime.creationdate" since it includes the time zone offset. */ -const parseMetadataCreationDate = (s: string | undefined) => { +const parseFFMetadataDate = (s: string | undefined) => { if (!s) return undefined; - const m = s.match(/(\+|-)\d+\.*\d+/g); - if (!m) { - log.warn(`Ignoring unparseable location string "${s}"`); + const d = parseMetadataDate(s); + if (!d) { + log.warn(`Ignoring unparseable video metadata date string "${s}"`); return undefined; } - const [latitude, longitude] = m.map(parseFloat); - if (!latitude || !longitude) { - log.warn(`Ignoring unparseable location string "${s}"`); + // While not strictly required, we retain the same behaviour as the image + // Exif parser of ignoring dates whose epoch is 0. + if (!d.timestamp) { + log.warn(`Ignoring zero video metadata date string "${s}"`); return undefined; } - return { latitude, longitude }; -}; - -const parseCreationTime = (creationTime: string | undefined) => { - let dateTime = null; - if (creationTime) { - dateTime = validateAndGetCreationUnixTimeInMicroSeconds( - new Date(creationTime), - ); - } - return dateTime; + return d; }; /** From 862495c29e2092c778a724df33397f6e313688fc Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 13:00:59 +0530 Subject: [PATCH 053/211] Up --- .../src/services/upload/uploadService.ts | 39 +++++++------------ .../new/photos/services/ffmpeg/index.ts | 19 +++------ 2 files changed, 18 insertions(+), 40 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 5eeb7ddbad..9150222f07 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -10,7 +10,6 @@ import { extractExif } from "@/new/photos/services/exif"; import * as ffmpeg from "@/new/photos/services/ffmpeg"; import type { UploadItem } from "@/new/photos/services/upload/types"; import { - NULL_LOCATION, RANDOM_PERCENTAGE_PROGRESS_FOR_PUT, UPLOAD_RESULT, } from "@/new/photos/services/upload/types"; @@ -23,7 +22,6 @@ import { type FilePublicMagicMetadataProps, } from "@/new/photos/types/file"; import { EncryptedMagicMetadata } from "@/new/photos/types/magicMetadata"; -import type { ParsedExtractedMetadata } from "@/new/photos/types/metadata"; import { detectFileTypeInfoFromChunk } from "@/new/photos/utils/detect-type"; import { readStream } from "@/new/photos/utils/native-stream"; import { ensure } from "@/utils/ensure"; @@ -747,18 +745,14 @@ const extractImageOrVideoMetadata = async ( const fileName = uploadItemFileName(uploadItem); const { fileType } = fileTypeInfo; - let extractedMetadata: ParsedExtractedMetadata; - if (fileType === FileType.image) { - extractedMetadata = - (await tryExtractImageMetadata( - uploadItem, - fileTypeInfo, - lastModifiedMs, - )) ?? NULL_EXTRACTED_METADATA; - } else if (fileType === FileType.video) { - extractedMetadata = - (await tryExtractVideoMetadata(uploadItem)) ?? - NULL_EXTRACTED_METADATA; + let parsedMetadata: ParsedMetadata; + if (fileType == FileType.image) { + parsedMetadata = await tryExtractImageMetadata( + uploadItem, + lastModifiedMs, + ); + } else if (fileType == FileType.video) { + parsedMetadata = await tryExtractVideoMetadata(uploadItem); } else { throw new Error(`Unexpected file type ${fileType} for ${uploadItem}`); } @@ -767,7 +761,7 @@ const extractImageOrVideoMetadata = async ( const modificationTime = lastModifiedMs * 1000; const creationTime = - extractedMetadata.creationTime ?? + parsedMetadata.creationTime ?? tryParseEpochMicrosecondsFromFileName(fileName) ?? modificationTime; @@ -775,15 +769,15 @@ const extractImageOrVideoMetadata = async ( title: fileName, creationTime, modificationTime, - latitude: extractedMetadata.location.latitude, - longitude: extractedMetadata.location.longitude, + latitude: parsedMetadata.location.latitude, + longitude: parsedMetadata.location.longitude, fileType, hash, }; const publicMagicMetadata: FilePublicMagicMetadataProps = { - w: extractedMetadata.width, - h: extractedMetadata.height, + w: parsedMetadata.width, + h: parsedMetadata.height, }; const takeoutMetadata = matchTakeoutMetadata( @@ -799,13 +793,6 @@ const extractImageOrVideoMetadata = async ( return { metadata, publicMagicMetadata }; }; -const NULL_EXTRACTED_METADATA: ParsedExtractedMetadata = { - location: { ...NULL_LOCATION }, - creationTime: null, - width: null, - height: null, -}; - const tryExtractImageMetadata = async ( uploadItem: UploadItem, lastModifiedMs: number, diff --git a/web/packages/new/photos/services/ffmpeg/index.ts b/web/packages/new/photos/services/ffmpeg/index.ts index 776764fc6f..e845726a7d 100644 --- a/web/packages/new/photos/services/ffmpeg/index.ts +++ b/web/packages/new/photos/services/ffmpeg/index.ts @@ -150,15 +150,6 @@ const extractVideoMetadataCommand = [ outputPathPlaceholder, ]; -enum MetadataTags { - CREATION_TIME = "creation_time", - APPLE_CONTENT_IDENTIFIER = "com.apple.quicktime.content.identifier", - APPLE_LIVE_PHOTO_IDENTIFIER = "com.apple.quicktime.live-photo.auto", - APPLE_CREATION_DATE = "com.apple.quicktime.creationdate", - APPLE_LOCATION_ISO = "com.apple.quicktime.location.ISO6709", - LOCATION = "location", -} - /** * Convert the output produced by running the FFmpeg * {@link extractVideoMetadataCommand} into a {@link ParsedMetadata}. @@ -179,17 +170,17 @@ const parseFFmpegExtractedMetadata = (ffmpegOutput: Uint8Array) => { const result: ParsedMetadata = {}; + const creationDate = + parseFFMetadataDate(kv.get("com.apple.quicktime.creationdate")) ?? + parseFFMetadataDate(kv.get("creation_time")); + if (creationDate) result.creationDate = creationDate; + const location = parseFFMetadataLocation( kv.get("com.apple.quicktime.location.ISO6709"), ) ?? parseFFMetadataLocation(kv.get("location")); if (location) result.location = location; - const creationDate = - parseFFMetadataDate(kv.get("com.apple.quicktime.creationdate")) ?? - parseFFMetadataDate(kv.get("creation_time")); - if (creationDate) result.creationDate = creationDate; - return result; }; From 9a60bf3ba605c08da8338451b9037a8c200dd585 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 13:06:46 +0530 Subject: [PATCH 054/211] Doc --- web/packages/media/file-metadata.ts | 36 ++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 02591a0374..c7a3b53444 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -59,10 +59,12 @@ import { FileType } from "./file-type"; * unchanged. */ export interface Metadata { - /** The "Ente" file type - image, video or live photo. */ + /** + * The "Ente" file type - image, video or live photo. + */ fileType: FileType; /** - * The file name. + * The name of the file (including its extension). * * See: [Note: File name for local EnteFile objects] */ @@ -78,11 +80,23 @@ export interface Metadata { * of the upload. */ creationTime: number; + /** + * The last modification time of the file (epoch microseconds). + */ modificationTime: number; + /** + * The latitude where the image or video was taken. + */ latitude: number; + /** + * The longitude where the image or video was taken. + */ longitude: number; - hasStaticThumbnail?: boolean; + /** + * A hash of the file's contents. + */ hash?: string; + hasStaticThumbnail?: boolean; imageHash?: string; videoHash?: string; localID?: number; @@ -159,7 +173,7 @@ export enum ItemVisibility { export interface PublicMagicMetadata { /** * A ISO 8601 date time string without a timezone, indicating the local time - * where the photo was taken. + * where the photo (or video) was taken. * * e.g. "2022-01-26T13:08:20". * @@ -188,6 +202,18 @@ export interface PublicMagicMetadata { * This field stores edits to the {@link title} {@link Metadata} field. */ editedName?: string; + /** + * The width of the photo (or video) in pixels. + * + * While this should usually be present, it is not guaranteed to be. + */ + w?: number; + /** + * The height of the photo (or video) in pixels, if available. + * + * While this should usually be present, it is not guaranteed to be. + */ + h?: number; /** * An arbitrary caption / description string that the user has added to the * file. @@ -197,8 +223,6 @@ export interface PublicMagicMetadata { */ caption?: string; uploaderName?: string; - w?: number; - h?: number; } /** From fe399762f5c5c55345327e69f99542b58d4ca897 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 13:22:15 +0530 Subject: [PATCH 055/211] Doc --- .../src/services/upload/uploadService.ts | 6 ++-- web/packages/media/file-metadata.ts | 30 ++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 9150222f07..787629aca3 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -3,7 +3,7 @@ import { basename } from "@/base/file"; import log from "@/base/log"; import { CustomErrorMessage } from "@/base/types/ipc"; import { hasFileHash } from "@/media/file"; -import type { Metadata, ParsedMetadata } from "@/media/file-metadata"; +import type { Metadata, ParsedMetadata, PublicMagicMetadata } from "@/media/file-metadata"; import { FileType, type FileTypeInfo } from "@/media/file-type"; import { encodeLivePhoto } from "@/media/live-photo"; import { extractExif } from "@/new/photos/services/exif"; @@ -766,16 +766,16 @@ const extractImageOrVideoMetadata = async ( modificationTime; const metadata: Metadata = { + fileType, title: fileName, creationTime, modificationTime, latitude: parsedMetadata.location.latitude, longitude: parsedMetadata.location.longitude, - fileType, hash, }; - const publicMagicMetadata: FilePublicMagicMetadataProps = { + const publicMagicMetadata: PublicMagicMetadata = { w: parsedMetadata.width, h: parsedMetadata.height, }; diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index c7a3b53444..58149f070d 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -72,12 +72,17 @@ export interface Metadata { /** * The time when this file was created (epoch microseconds). * - * For photos (and images in general), this is our best attempt (using Exif - * and other metadata, or deducing it from file name for screenshots without - * any embedded metadata) at detecting the time when the photo was taken. + * This is our best attempt at detecting the time when the photo or live + * photo or video was taken. * - * If nothing can be found, then it is set to the current time at the time - * of the upload. + * - We first try to obtain this from metadata, using Exif and other + * metadata for images and FFmpeg-extracted metadata for video. + * + * - If no suitable metadata is available, then we try to deduce it from + * file name (e.g. for screenshots without any embedded metadata). + * + * - If nothing can be found, then it is set to the current time at the + * time of the upload. */ creationTime: number; /** @@ -85,20 +90,29 @@ export interface Metadata { */ modificationTime: number; /** - * The latitude where the image or video was taken. + * The latitude where the file was taken. */ latitude: number; /** - * The longitude where the image or video was taken. + * The longitude where the file was taken. */ longitude: number; /** * A hash of the file's contents. + * + * It is only valid for images and videos. For live photos, see + * {@link imageHash} and {@link videoHash}. */ hash?: string; - hasStaticThumbnail?: boolean; + /** + * The hash of the image component of a live photo. + */ imageHash?: string; + /** + * The hash of the video component of a live photo. + */ videoHash?: string; + hasStaticThumbnail?: boolean; localID?: number; version?: number; deviceFolder?: string; From 920b4e6823b4a45fb920e95b4b2c6243cca12b45 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 13:55:18 +0530 Subject: [PATCH 056/211] Unnull --- .../src/services/upload/uploadService.ts | 21 ++++++++++++------- web/packages/media/file-metadata.ts | 4 ++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 787629aca3..4840db5834 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -3,7 +3,11 @@ import { basename } from "@/base/file"; import log from "@/base/log"; import { CustomErrorMessage } from "@/base/types/ipc"; import { hasFileHash } from "@/media/file"; -import type { Metadata, ParsedMetadata, PublicMagicMetadata } from "@/media/file-metadata"; +import type { + Metadata, + ParsedMetadata, + PublicMagicMetadata, +} from "@/media/file-metadata"; import { FileType, type FileTypeInfo } from "@/media/file-type"; import { encodeLivePhoto } from "@/media/live-photo"; import { extractExif } from "@/new/photos/services/exif"; @@ -765,20 +769,23 @@ const extractImageOrVideoMetadata = async ( tryParseEpochMicrosecondsFromFileName(fileName) ?? modificationTime; + const { width: w, height: h, location } = parsedMetadata; + const metadata: Metadata = { fileType, title: fileName, creationTime, modificationTime, - latitude: parsedMetadata.location.latitude, - longitude: parsedMetadata.location.longitude, hash, }; + if (location) { + metadata.latitude = location.latitude; + metadata.longitude = location.longitude; + } - const publicMagicMetadata: PublicMagicMetadata = { - w: parsedMetadata.width, - h: parsedMetadata.height, - }; + const publicMagicMetadata: PublicMagicMetadata = {}; + if (w) publicMagicMetadata.w = w; + if (h) publicMagicMetadata.h = h; const takeoutMetadata = matchTakeoutMetadata( fileName, diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 58149f070d..a58202819b 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -92,11 +92,11 @@ export interface Metadata { /** * The latitude where the file was taken. */ - latitude: number; + latitude?: number; /** * The longitude where the file was taken. */ - longitude: number; + longitude?: number; /** * A hash of the file's contents. * From 1db7bf2902f6664e48d29a0145139badf04c5449 Mon Sep 17 00:00:00 2001 From: Shamshid Date: Wed, 7 Aug 2024 12:37:14 +0400 Subject: [PATCH 057/211] [Auth] Add 3 new icons (#2627) ## Description Add icons for: - [Booking.com](http://booking.com/) - [Blockchain.com](https://blockchain.com/) - [BitOasis](https://bitoasis.net/) --- auth/assets/custom-icons/_data/custom-icons.json | 15 +++++++++++++++ auth/assets/custom-icons/icons/bitoasis.svg | 3 +++ auth/assets/custom-icons/icons/blockchain.svg | 5 +++++ auth/assets/custom-icons/icons/booking.svg | 4 ++++ 4 files changed, 27 insertions(+) create mode 100644 auth/assets/custom-icons/icons/bitoasis.svg create mode 100644 auth/assets/custom-icons/icons/blockchain.svg create mode 100644 auth/assets/custom-icons/icons/booking.svg diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index e986cc7b61..80f81b2887 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -40,6 +40,9 @@ { "title": "BitMEX" }, + { + "title": "BitOasis" + }, { "title": "BitSkins" }, @@ -60,6 +63,14 @@ "Bloom Host Billing" ] }, + { + "title": "Blockchain.com", + "altNames": [ + "blockchain.com Wallet", + "blockchain.com Exchange" + ], + "slug": "blockchain" + }, { "title": "BorgBase", "altNames": [ @@ -67,6 +78,10 @@ ], "slug": "BorgBase" }, + { + "title": "Booking.com", + "slug": "booking" + }, { "title": "Brave Creators", "slug": "brave_creators" diff --git a/auth/assets/custom-icons/icons/bitoasis.svg b/auth/assets/custom-icons/icons/bitoasis.svg new file mode 100644 index 0000000000..b620c7c453 --- /dev/null +++ b/auth/assets/custom-icons/icons/bitoasis.svg @@ -0,0 +1,3 @@ + + + diff --git a/auth/assets/custom-icons/icons/blockchain.svg b/auth/assets/custom-icons/icons/blockchain.svg new file mode 100644 index 0000000000..82667fa7a9 --- /dev/null +++ b/auth/assets/custom-icons/icons/blockchain.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/auth/assets/custom-icons/icons/booking.svg b/auth/assets/custom-icons/icons/booking.svg new file mode 100644 index 0000000000..419b0d98b7 --- /dev/null +++ b/auth/assets/custom-icons/icons/booking.svg @@ -0,0 +1,4 @@ + + + + From 0a3182be53871958cc0986bd4b0436d9ab5288ce Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 14:12:05 +0530 Subject: [PATCH 058/211] Not deprecated --- .../components/PhotoViewer/FileInfo/index.tsx | 16 +++++++++------- web/apps/photos/src/services/upload/takeout.ts | 9 ++++++--- web/packages/media/file-metadata.ts | 16 +++++++++------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index 25ce1d61b1..d724274640 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -383,13 +383,15 @@ export const CreationTime: React.FC = ({ // [Note: Don't modify offsetTime when editing date via picker] // // Use the updated date time (both in its canonical dateTime - // form, and also as the legacy timestamp). But don't use the - // offset. The offset here will be the offset of the computer - // where this user is making this edit, not the offset of the - // place where the photo was taken. In a future iteration of the - // date time editor, we can provide functionality for the user - // to edit the associated offset, but right now it is not even - // surfaced, so don't also potentially overwrite it. + // form, and also as in the epoch timestamp), but don't use the + // offset. + // + // The offset here will be the offset of the computer where this + // user is making this edit, not the offset of the place where + // the photo was taken. In a future iteration of the date time + // editor, we can provide functionality for the user to edit the + // associated offset, but right now it is not even surfaced, so + // don't also potentially overwrite it. const { dateTime, timestamp } = pickedTime; if (timestamp == originalDate.getTime()) { // Same as before. diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index cdb2880d8f..f0d97b2161 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -114,26 +114,29 @@ const parseMetadataJSONText = (text: string) => { const parsedMetadataJSON = { ...NULL_PARSED_METADATA_JSON }; + // The metadata provided by Google does not include the time zone where the + // photo was taken, it only has an epoch seconds value. if ( metadataJSON["photoTakenTime"] && metadataJSON["photoTakenTime"]["timestamp"] ) { parsedMetadataJSON.creationTime = - metadataJSON["photoTakenTime"]["timestamp"] * 1000000; + metadataJSON["photoTakenTime"]["timestamp"] * 1e6; } else if ( metadataJSON["creationTime"] && metadataJSON["creationTime"]["timestamp"] ) { parsedMetadataJSON.creationTime = - metadataJSON["creationTime"]["timestamp"] * 1000000; + metadataJSON["creationTime"]["timestamp"] * 1e6; } if ( metadataJSON["modificationTime"] && metadataJSON["modificationTime"]["timestamp"] ) { parsedMetadataJSON.modificationTime = - metadataJSON["modificationTime"]["timestamp"] * 1000000; + metadataJSON["modificationTime"]["timestamp"] * 1e6; } + let locationData: Location = { ...NULL_LOCATION }; if ( metadataJSON["geoData"] && diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index a58202819b..3e75dac66f 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -636,10 +636,11 @@ export interface ParsedMetadata { * offset, its siblings photos might not. The only way to retain their * comparability is to treat them all the "time zone where the photo was taken". * - * Finally, while this is all great, we still have existing code that deals with - * UTC timestamps. So we also retain the existing `creationTime` UTC timestamp, - * but this should be considered deprecated, and over time we should move - * towards using the `dateTime` string. + * All this is good, but we still need to retain the existing `creationTime` UTC + * epoch timestamp because in some cases when importing photos from other + * providers, that's all we get. We could try and convert that to a date/time + * string too, but since we anyways need to handle existing code that deals with + * epoch timestamps, we retain them as they were provided. */ export interface ParsedMetadataDate { /** @@ -760,8 +761,9 @@ export const parseMetadataDate = ( const dropLast = (s: string) => (s ? s.substring(0, s.length - 1) : s); /** - * Return a date that can be used on the UI from a {@link ParsedMetadataDate}, - * or its {@link dateTime} component, or the legacy epoch timestamps. + * Return a date that can be used on the UI by constructing it from a + * {@link ParsedMetadataDate}, or its {@link dateTime} component, or a UTC epoch + * timestamp. * * These dates are all hypothetically in the timezone of the place where the * photo was taken. Different photos might've been taken in different timezones, @@ -791,7 +793,7 @@ export const toUIDate = (dateLike: ParsedMetadataDate | string | number) => { // `ParsedMetadataDate.dateTime`. return new Date(dateLike); case "number": - // A legacy epoch microseconds value. + // A UTC epoch microseconds value. return new Date(dateLike / 1000); } }; From 139d3b99a116ddb8a565d585164bef4f6ae011bb Mon Sep 17 00:00:00 2001 From: dnred <174188760+dnred@users.noreply.github.com> Date: Wed, 7 Aug 2024 11:04:01 +0200 Subject: [PATCH 059/211] Change Auth mobile apps' names to "Ente Auth" (#2622) ## Description Changed the name of the Auth mobile app to "Ente Auth" on both Android and iOS to make it consistent with the naming of Ente Photos and to also make it consistent on both platforms. --- auth/android/app/src/main/AndroidManifest.xml | 2 +- auth/ios/Runner/Info.plist | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auth/android/app/src/main/AndroidManifest.xml b/auth/android/app/src/main/AndroidManifest.xml index 7c7a8ba5f0..518f1df65a 100644 --- a/auth/android/app/src/main/AndroidManifest.xml +++ b/auth/android/app/src/main/AndroidManifest.xml @@ -1,7 +1,7 @@ CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleDisplayName - auth + Ente Auth CFBundleExecutable $(EXECUTABLE_NAME) CFBundleIdentifier From 1fdeebed28fddd4e181ea0f88cc5d9ef78a5d8a1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 14:37:12 +0530 Subject: [PATCH 060/211] Integrate --- .../photos/src/services/upload/takeout.ts | 43 ++++++++++--------- .../src/services/upload/uploadService.ts | 26 +++++------ 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/services/upload/takeout.ts b/web/apps/photos/src/services/upload/takeout.ts index f0d97b2161..3ece43a4c0 100644 --- a/web/apps/photos/src/services/upload/takeout.ts +++ b/web/apps/photos/src/services/upload/takeout.ts @@ -1,18 +1,22 @@ -/** @file Dealing with the JSON metadata in Google Takeouts */ +/** @file Dealing with the JSON metadata sidecar files */ import { ensureElectron } from "@/base/electron"; import { nameAndExtension } from "@/base/file"; import log from "@/base/log"; import type { UploadItem } from "@/new/photos/services/upload/types"; -import { NULL_LOCATION } from "@/new/photos/services/upload/types"; -import type { Location } from "@/new/photos/types/metadata"; import { readStream } from "@/new/photos/utils/native-stream"; +/** + * The data we read from the JSON metadata sidecar files. + * + * Originally these were used to read the JSON metadata sidecar files present in + * a Google Takeout. However, during our own export, we also write out files + * with a similar structure. + */ export interface ParsedMetadataJSON { - creationTime: number; - modificationTime: number; - latitude: number; - longitude: number; + creationTime?: number; + modificationTime?: number; + location?: { latitude: number; longitude: number }; } export const MAX_FILE_NAME_LENGTH_GOOGLE_EXPORT = 46; @@ -100,19 +104,13 @@ const uploadItemText = async (uploadItem: UploadItem) => { } }; -const NULL_PARSED_METADATA_JSON: ParsedMetadataJSON = { - creationTime: null, - modificationTime: null, - ...NULL_LOCATION, -}; - const parseMetadataJSONText = (text: string) => { const metadataJSON: object = JSON.parse(text); if (!metadataJSON) { return undefined; } - const parsedMetadataJSON = { ...NULL_PARSED_METADATA_JSON }; + const parsedMetadataJSON: ParsedMetadataJSON = {}; // The metadata provided by Google does not include the time zone where the // photo was taken, it only has an epoch seconds value. @@ -129,6 +127,7 @@ const parseMetadataJSONText = (text: string) => { parsedMetadataJSON.creationTime = metadataJSON["creationTime"]["timestamp"] * 1e6; } + if ( metadataJSON["modificationTime"] && metadataJSON["modificationTime"]["timestamp"] @@ -137,24 +136,26 @@ const parseMetadataJSONText = (text: string) => { metadataJSON["modificationTime"]["timestamp"] * 1e6; } - let locationData: Location = { ...NULL_LOCATION }; if ( metadataJSON["geoData"] && (metadataJSON["geoData"]["latitude"] !== 0.0 || metadataJSON["geoData"]["longitude"] !== 0.0) ) { - locationData = metadataJSON["geoData"]; + parsedMetadataJSON.location = { + latitude: metadataJSON["geoData"]["latitude"], + longitude: metadataJSON["geoData"]["longitude"], + }; } else if ( metadataJSON["geoDataExif"] && (metadataJSON["geoDataExif"]["latitude"] !== 0.0 || metadataJSON["geoDataExif"]["longitude"] !== 0.0) ) { - locationData = metadataJSON["geoDataExif"]; - } - if (locationData !== null) { - parsedMetadataJSON.latitude = locationData.latitude; - parsedMetadataJSON.longitude = locationData.longitude; + parsedMetadataJSON.location = { + latitude: metadataJSON["geoDataExif"]["latitude"], + longitude: metadataJSON["geoDataExif"]["longitude"], + }; } + return parsedMetadataJSON; }; diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index 4840db5834..aabdf92e2e 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -763,14 +763,20 @@ const extractImageOrVideoMetadata = async ( const hash = await computeHash(uploadItem, worker); - const modificationTime = lastModifiedMs * 1000; + const parsedMetadataJSON = matchTakeoutMetadata( + fileName, + collectionID, + parsedMetadataJSONMap, + ); + + const modificationTime = + parsedMetadataJSON.modificationTime ?? lastModifiedMs * 1000; const creationTime = + parsedMetadataJSON.creationTime ?? parsedMetadata.creationTime ?? tryParseEpochMicrosecondsFromFileName(fileName) ?? modificationTime; - const { width: w, height: h, location } = parsedMetadata; - const metadata: Metadata = { fileType, title: fileName, @@ -778,25 +784,19 @@ const extractImageOrVideoMetadata = async ( modificationTime, hash, }; + + const location = parsedMetadataJSON.location ?? parsedMetadata.location; if (location) { metadata.latitude = location.latitude; metadata.longitude = location.longitude; } + const { width: w, height: h } = parsedMetadata; + const publicMagicMetadata: PublicMagicMetadata = {}; if (w) publicMagicMetadata.w = w; if (h) publicMagicMetadata.h = h; - const takeoutMetadata = matchTakeoutMetadata( - fileName, - collectionID, - parsedMetadataJSONMap, - ); - - if (takeoutMetadata) - for (const [key, value] of Object.entries(takeoutMetadata)) - if (value) metadata[key] = value; - return { metadata, publicMagicMetadata }; }; From 0b279111ddb295b6ee83099b7d120849a75ed17f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 15:01:26 +0530 Subject: [PATCH 061/211] Use during uploads --- .../src/services/upload/uploadService.ts | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index aabdf92e2e..a7b0f5c477 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -769,13 +769,23 @@ const extractImageOrVideoMetadata = async ( parsedMetadataJSONMap, ); + const publicMagicMetadata: PublicMagicMetadata = {}; + const modificationTime = - parsedMetadataJSON.modificationTime ?? lastModifiedMs * 1000; - const creationTime = - parsedMetadataJSON.creationTime ?? - parsedMetadata.creationTime ?? - tryParseEpochMicrosecondsFromFileName(fileName) ?? - modificationTime; + parsedMetadataJSON?.modificationTime ?? lastModifiedMs * 1000; + + let creationTime: number; + if (parsedMetadataJSON?.creationTime) { + creationTime = parsedMetadataJSON.creationTime; + } else if (parsedMetadata.creationDate) { + const { dateTime, offset, timestamp } = parsedMetadata.creationDate; + creationTime = timestamp; + publicMagicMetadata.dateTime = dateTime; + if (offset) publicMagicMetadata.offsetTime = offset; + } else { + creationTime = + tryParseEpochMicrosecondsFromFileName(fileName) ?? modificationTime; + } const metadata: Metadata = { fileType, @@ -785,15 +795,13 @@ const extractImageOrVideoMetadata = async ( hash, }; - const location = parsedMetadataJSON.location ?? parsedMetadata.location; + const location = parsedMetadataJSON?.location ?? parsedMetadata.location; if (location) { metadata.latitude = location.latitude; metadata.longitude = location.longitude; } const { width: w, height: h } = parsedMetadata; - - const publicMagicMetadata: PublicMagicMetadata = {}; if (w) publicMagicMetadata.w = w; if (h) publicMagicMetadata.h = h; From 5f14057b658683177f616571d8d8130419348314 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:34:21 +0530 Subject: [PATCH 062/211] Update schema to add in-flight list of regions --- server/migrations/89_file_data_table.up.sql | 34 ++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/server/migrations/89_file_data_table.up.sql b/server/migrations/89_file_data_table.up.sql index 49c9704ee6..36d66587a6 100644 --- a/server/migrations/89_file_data_table.up.sql +++ b/server/migrations/89_file_data_table.up.sql @@ -12,6 +12,7 @@ CREATE TABLE IF NOT EXISTS file_data replicated_buckets s3region[] NOT NULL DEFAULT '{}', -- following field contains list of buckets from where we need to delete the data as the given data_type will not longer be persisted in that dc delete_from_buckets s3region[] NOT NULL DEFAULT '{}', + inflight_rep_buckets s3region[] NOT NULL DEFAULT '{}', pending_sync BOOLEAN NOT NULL DEFAULT false, is_deleted BOOLEAN NOT NULL DEFAULT false, last_sync_time BIGINT NOT NULL DEFAULT 0, @@ -20,29 +21,28 @@ CREATE TABLE IF NOT EXISTS file_data PRIMARY KEY (file_id, data_type) ); - -- Add index for user_id and data_type for efficient querying CREATE INDEX idx_file_data_user_type_deleted ON file_data (user_id, data_type, is_deleted) INCLUDE (file_id, size); CREATE OR REPLACE FUNCTION ensure_no_common_entries() RETURNS TRIGGER AS $$ +DECLARE + all_buckets s3region[]; + duplicate_buckets s3region[]; BEGIN - -- Check for common entries between latest_bucket and replicated_buckets - IF NEW.latest_bucket = ANY(NEW.replicated_buckets) THEN - RAISE EXCEPTION 'latest_bucket and replicated_buckets have common entries'; - END IF; - - -- Check for common entries between latest_bucket and delete_from_buckets - IF NEW.latest_bucket = ANY(NEW.delete_from_buckets) THEN - RAISE EXCEPTION 'latest_bucket and delete_from_buckets have common entries'; - END IF; - - -- Check for common entries between replicated_buckets and delete_from_buckets - IF EXISTS ( - SELECT 1 FROM unnest(NEW.replicated_buckets) AS rb - WHERE rb = ANY(NEW.delete_from_buckets) - ) THEN - RAISE EXCEPTION 'replicated_buckets and delete_from_buckets have common entries'; + -- Combine all bucket IDs into a single array + all_buckets := ARRAY[NEW.latest_bucket] || NEW.replicated_buckets || NEW.delete_from_buckets || NEW.inflight_rep_buckets; + + -- Find duplicate bucket IDs + SELECT ARRAY_AGG(DISTINCT bucket) INTO duplicate_buckets + FROM unnest(all_buckets) bucket + GROUP BY bucket + HAVING COUNT(*) > 1; + + -- If duplicates exist, raise an exception with details + IF ARRAY_LENGTH(duplicate_buckets, 1) > 0 THEN + RAISE EXCEPTION 'Duplicate bucket IDs found: %. Latest: %, Replicated: %, To Delete: %, Inflight: %', + duplicate_buckets, NEW.latest_bucket, NEW.replicated_buckets, NEW.delete_from_buckets, NEW.inflight_rep_buckets; END IF; RETURN NEW; From 527dfc3721fed01f88ce4d15cf38f331ab162f14 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 15:29:02 +0530 Subject: [PATCH 063/211] Remove exifr --- web/apps/photos/package.json | 1 - .../src/services/locationSearchService.ts | 3 +- web/apps/photos/src/services/upload/date.ts | 16 +- web/apps/photos/src/types/entity.ts | 7 +- web/packages/new/photos/services/exif.ts | 56 --- web/packages/new/photos/services/ml/worker.ts | 9 +- .../new/photos/services/upload/types.ts | 3 - web/packages/new/photos/types/metadata.ts | 11 - web/packages/shared/time/index.ts | 15 - web/packages/shared/utils/exif-old.ts | 340 ------------------ web/yarn.lock | 5 - 11 files changed, 22 insertions(+), 444 deletions(-) delete mode 100644 web/packages/new/photos/types/metadata.ts delete mode 100644 web/packages/shared/utils/exif-old.ts diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json index c2b0d85da9..f8b043adce 100644 --- a/web/apps/photos/package.json +++ b/web/apps/photos/package.json @@ -15,7 +15,6 @@ "bs58": "^5.0.0", "chrono-node": "^2.2.6", "debounce": "^2.0.0", - "exifr": "^7.1.3", "exifreader": "^4", "fast-srp-hap": "^2.0.4", "ffmpeg-wasm": "file:./thirdparty/ffmpeg-wasm", diff --git a/web/apps/photos/src/services/locationSearchService.ts b/web/apps/photos/src/services/locationSearchService.ts index d28e8190da..30a36e5e04 100644 --- a/web/apps/photos/src/services/locationSearchService.ts +++ b/web/apps/photos/src/services/locationSearchService.ts @@ -1,6 +1,5 @@ import log from "@/base/log"; -import type { Location } from "@/new/photos/types/metadata"; -import type { LocationTagData } from "types/entity"; +import type { Location, LocationTagData } from "types/entity"; export interface City { city: string; diff --git a/web/apps/photos/src/services/upload/date.ts b/web/apps/photos/src/services/upload/date.ts index d70e00b5ef..d18aeeb5eb 100644 --- a/web/apps/photos/src/services/upload/date.ts +++ b/web/apps/photos/src/services/upload/date.ts @@ -1,5 +1,4 @@ import log from "@/base/log"; -import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; /** * Try to extract a date (as epoch microseconds) from a file name by matching it @@ -41,6 +40,21 @@ export const tryParseEpochMicrosecondsFromFileName = ( } }; +export function validateAndGetCreationUnixTimeInMicroSeconds(dateTime: Date) { + if (!dateTime || isNaN(dateTime.getTime())) { + return null; + } + const unixTime = dateTime.getTime() * 1000; + //ignoring dateTimeString = "0000:00:00 00:00:00" + if (unixTime === Date.UTC(0, 0, 0, 0, 0, 0, 0) || unixTime === 0) { + return null; + } else if (unixTime > Date.now() * 1000) { + return null; + } else { + return unixTime; + } +} + interface DateComponent { year: T; month: T; diff --git a/web/apps/photos/src/types/entity.ts b/web/apps/photos/src/types/entity.ts index 0f22973d21..4371562cc8 100644 --- a/web/apps/photos/src/types/entity.ts +++ b/web/apps/photos/src/types/entity.ts @@ -1,5 +1,3 @@ -import { Location } from "@/new/photos/types/metadata"; - export enum EntityType { LOCATION_TAG = "location", } @@ -27,6 +25,11 @@ export interface EncryptedEntity { userID: number; } +export interface Location { + latitude: number | null; + longitude: number | null; +} + export interface LocationTagData { name: string; radius: number; diff --git a/web/packages/new/photos/services/exif.ts b/web/packages/new/photos/services/exif.ts index d559b9750a..5e7a7ea8ec 100644 --- a/web/packages/new/photos/services/exif.ts +++ b/web/packages/new/photos/services/exif.ts @@ -1,66 +1,10 @@ -import { nameAndExtension } from "@/base/file"; import log from "@/base/log"; import { parseMetadataDate, type ParsedMetadata, type ParsedMetadataDate, } from "@/media/file-metadata"; -import { FileType } from "@/media/file-type"; -import { parseImageMetadata } from "@ente/shared/utils/exif-old"; import ExifReader from "exifreader"; -import type { EnteFile } from "../types/file"; -import type { ParsedExtractedMetadata } from "../types/metadata"; - -const cmpTsEq = (a: number | undefined | null, b: number | undefined) => { - if (!a && !b) return true; - if (!a || !b) return false; - if (a == b) return true; - if (Math.floor(a / 1e6) == Math.floor(b / 1e6)) return true; - return false; -}; - -export const cmpNewLib = ( - oldLib: ParsedExtractedMetadata, - newLib: ParsedMetadata, -) => { - const logM = (r: string) => - log.info("[exif]", r, JSON.stringify({ old: oldLib, new: newLib })); - if ( - cmpTsEq(oldLib.creationTime, newLib.creationDate?.timestamp) && - oldLib.location.latitude == newLib.location?.latitude && - oldLib.location.longitude == newLib.location?.longitude - ) { - if ( - oldLib.width == newLib.width && - oldLib.height == newLib.height && - oldLib.creationTime == newLib.creationDate?.timestamp - ) - logM("exact match"); - else logM("enhanced match"); - log.debug(() => ["exif/cmp", { oldLib, newLib }]); - } else { - logM("potential mismatch ❗️🚩"); - } -}; - -export const cmpNewLib2 = async ( - enteFile: EnteFile, - blob: Blob, - _exif: unknown, -) => { - const [, ext] = nameAndExtension(enteFile.metadata.title); - const oldLib = await parseImageMetadata( - new File([blob], enteFile.metadata.title), - { - fileType: FileType.image, - extension: ext ?? "", - }, - ); - // cast is fine here, this is just temporary debugging code. - const rawExif = _exif as RawExifTags; - const newLib = parseExif(rawExif); - cmpNewLib(oldLib, newLib); -}; /** * Extract Exif and other metadata from the given file. diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index e012ebab16..2fe408f9bb 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -11,7 +11,7 @@ import { wait } from "@/utils/promise"; import { DOMParser } from "@xmldom/xmldom"; import { expose, wrap } from "comlink"; import downloadManager from "../download"; -import { cmpNewLib2, extractRawExif, type RawExifTags } from "../exif"; +import { extractRawExif, type RawExifTags } from "../exif"; import { getAllLocalFiles, getLocalTrashedFiles } from "../files"; import type { UploadItem } from "../upload/types"; import { @@ -518,13 +518,6 @@ const index = async ( throw e; } - try { - if (originalImageBlob && exif) - await cmpNewLib2(enteFile, originalImageBlob, exif); - } catch (e) { - log.warn(`Skipping exif cmp for ${f}`, e); - } - log.debug(() => { const ms = Date.now() - startTime; const msg = []; diff --git a/web/packages/new/photos/services/upload/types.ts b/web/packages/new/photos/services/upload/types.ts index d8d8517803..094e6704eb 100644 --- a/web/packages/new/photos/services/upload/types.ts +++ b/web/packages/new/photos/services/upload/types.ts @@ -1,5 +1,4 @@ import type { ZipItem } from "@/base/types/ipc"; -import type { Location } from "../../types/metadata"; /** * An item to upload is one of the following: @@ -59,8 +58,6 @@ export const toDataOrPathOrZipEntry = (desktopUploadItem: DesktopUploadItem) => export const RANDOM_PERCENTAGE_PROGRESS_FOR_PUT = () => 90 + 10 * Math.random(); -export const NULL_LOCATION: Location = { latitude: null, longitude: null }; - export enum UPLOAD_STAGES { START, READING_GOOGLE_METADATA_FILES, diff --git a/web/packages/new/photos/types/metadata.ts b/web/packages/new/photos/types/metadata.ts deleted file mode 100644 index 8c7ee8088e..0000000000 --- a/web/packages/new/photos/types/metadata.ts +++ /dev/null @@ -1,11 +0,0 @@ -export interface Location { - latitude: number | null; - longitude: number | null; -} - -export interface ParsedExtractedMetadata { - location: Location; - creationTime: number | null; - width: number | null; - height: number | null; -} diff --git a/web/packages/shared/time/index.ts b/web/packages/shared/time/index.ts index 87e1d9648b..52c9c499bd 100644 --- a/web/packages/shared/time/index.ts +++ b/web/packages/shared/time/index.ts @@ -22,21 +22,6 @@ export function getUnixTimeInMicroSecondsWithDelta(delta: TimeDelta): number { return currentDate.getTime() * 1000; } -export function validateAndGetCreationUnixTimeInMicroSeconds(dateTime: Date) { - if (!dateTime || isNaN(dateTime.getTime())) { - return null; - } - const unixTime = dateTime.getTime() * 1000; - //ignoring dateTimeString = "0000:00:00 00:00:00" - if (unixTime === Date.UTC(0, 0, 0, 0, 0, 0, 0) || unixTime === 0) { - return null; - } else if (unixTime > Date.now() * 1000) { - return null; - } else { - return unixTime; - } -} - function _addDays(date: Date, days: number): Date { const result = new Date(date); result.setDate(date.getDate() + days); diff --git a/web/packages/shared/utils/exif-old.ts b/web/packages/shared/utils/exif-old.ts deleted file mode 100644 index b0041edc43..0000000000 --- a/web/packages/shared/utils/exif-old.ts +++ /dev/null @@ -1,340 +0,0 @@ -// The code in this file is deprecated and meant to be deleted. -// -// eslint-disable-next-line @typescript-eslint/ban-ts-comment -// @ts-nocheck - -import log from "@/base/log"; -import { type FileTypeInfo } from "@/media/file-type"; -import { NULL_LOCATION } from "@/new/photos/services/upload/types"; -import type { - Location, - ParsedExtractedMetadata, -} from "@/new/photos/types/metadata"; -import { validateAndGetCreationUnixTimeInMicroSeconds } from "@ente/shared/time"; -import exifr from "exifr"; - -type ParsedEXIFData = Record & - Partial<{ - DateTimeOriginal: Date; - CreateDate: Date; - ModifyDate: Date; - DateCreated: Date; - MetadataDate: Date; - latitude: number; - longitude: number; - imageWidth: number; - imageHeight: number; - }>; - -type RawEXIFData = Record & - Partial<{ - DateTimeOriginal: string; - CreateDate: string; - ModifyDate: string; - DateCreated: string; - MetadataDate: string; - GPSLatitude: number[]; - GPSLongitude: number[]; - GPSLatitudeRef: string; - GPSLongitudeRef: string; - ImageWidth: number; - ImageHeight: number; - }>; - -const exifTagsNeededForParsingImageMetadata = [ - "DateTimeOriginal", - "CreateDate", - "ModifyDate", - "GPSLatitude", - "GPSLongitude", - "GPSLatitudeRef", - "GPSLongitudeRef", - "DateCreated", - "ExifImageWidth", - "ExifImageHeight", - "ImageWidth", - "ImageHeight", - "PixelXDimension", - "PixelYDimension", - "MetadataDate", -]; - -/** - * Read Exif data from an image {@link file} and use that to construct and - * return an {@link ParsedExtractedMetadata}. - * - * This function is tailored for use when we upload files. - */ -export const parseImageMetadata = async ( - file: File, - fileTypeInfo: FileTypeInfo, -): Promise => { - const exifData = await getParsedExifData( - file, - fileTypeInfo, - exifTagsNeededForParsingImageMetadata, - ); - - // TODO: Exif- remove me. - log.debug(() => ["exif/old", exifData]); - return { - location: getEXIFLocation(exifData), - creationTime: getEXIFTime(exifData), - width: exifData?.imageWidth ?? null, - height: exifData?.imageHeight ?? null, - }; -}; - -export async function getParsedExifData( - receivedFile: File, - { extension }: FileTypeInfo, - tags?: string[], -): Promise { - const exifLessFormats = ["gif", "bmp"]; - const exifrUnsupportedFileFormatMessage = "Unknown file format"; - - try { - if (exifLessFormats.includes(extension)) return null; - - const exifData: RawEXIFData = await exifr.parse(receivedFile, { - reviveValues: false, - tiff: true, - xmp: true, - icc: true, - iptc: true, - jfif: true, - ihdr: true, - }); - if (!exifData) { - return null; - } - const filteredExifData = tags - ? Object.fromEntries( - Object.entries(exifData).filter(([key]) => - tags.includes(key), - ), - ) - : exifData; - return parseExifData(filteredExifData); - } catch (e) { - if (e.message == exifrUnsupportedFileFormatMessage) { - log.error(`EXIFR does not support ${extension} files`, e); - return undefined; - } else { - log.error(`Failed to parse Exif data for a ${extension} file`, e); - throw e; - } - } -} - -function parseExifData(exifData: RawEXIFData): ParsedEXIFData { - if (!exifData) { - return null; - } - const { - DateTimeOriginal, - CreateDate, - ModifyDate, - DateCreated, - ImageHeight, - ImageWidth, - ExifImageHeight, - ExifImageWidth, - PixelXDimension, - PixelYDimension, - MetadataDate, - ...rest - } = exifData; - const parsedExif: ParsedEXIFData = { ...rest }; - if (DateTimeOriginal) { - parsedExif.DateTimeOriginal = parseEXIFDate(exifData.DateTimeOriginal); - } - if (CreateDate) { - parsedExif.CreateDate = parseEXIFDate(exifData.CreateDate); - } - if (ModifyDate) { - parsedExif.ModifyDate = parseEXIFDate(exifData.ModifyDate); - } - if (DateCreated) { - parsedExif.DateCreated = parseEXIFDate(exifData.DateCreated); - } - if (MetadataDate) { - parsedExif.MetadataDate = parseEXIFDate(exifData.MetadataDate); - } - if (exifData.GPSLatitude && exifData.GPSLongitude) { - const parsedLocation = parseEXIFLocation( - exifData.GPSLatitude, - exifData.GPSLatitudeRef, - exifData.GPSLongitude, - exifData.GPSLongitudeRef, - ); - parsedExif.latitude = parsedLocation.latitude; - parsedExif.longitude = parsedLocation.longitude; - } - if (ImageWidth && ImageHeight) { - if (typeof ImageWidth === "number" && typeof ImageHeight === "number") { - parsedExif.imageWidth = ImageWidth; - parsedExif.imageHeight = ImageHeight; - } else { - log.warn("Exif: Ignoring non-numeric ImageWidth or ImageHeight"); - } - } else if (ExifImageWidth && ExifImageHeight) { - if ( - typeof ExifImageWidth === "number" && - typeof ExifImageHeight === "number" - ) { - parsedExif.imageWidth = ExifImageWidth; - parsedExif.imageHeight = ExifImageHeight; - } else { - log.warn( - "Exif: Ignoring non-numeric ExifImageWidth or ExifImageHeight", - ); - } - } else if (PixelXDimension && PixelYDimension) { - if ( - typeof PixelXDimension === "number" && - typeof PixelYDimension === "number" - ) { - parsedExif.imageWidth = PixelXDimension; - parsedExif.imageHeight = PixelYDimension; - } else { - log.warn( - "Exif: Ignoring non-numeric PixelXDimension or PixelYDimension", - ); - } - } - return parsedExif; -} - -function parseEXIFDate(dateTimeString: string) { - try { - if (typeof dateTimeString !== "string" || dateTimeString === "") { - throw new Error("Invalid date string"); - } - - // Check and parse date in the format YYYYMMDD - if (dateTimeString.length === 8) { - const year = Number(dateTimeString.slice(0, 4)); - const month = Number(dateTimeString.slice(4, 6)); - const day = Number(dateTimeString.slice(6, 8)); - if ( - !Number.isNaN(year) && - !Number.isNaN(month) && - !Number.isNaN(day) - ) { - const date = new Date(year, month - 1, day); - if (!Number.isNaN(+date)) { - return date; - } - } - } - const [year, month, day, hour, minute, second] = dateTimeString - .match(/\d+/g) - .map(Number); - - if ( - typeof year === "undefined" || - Number.isNaN(year) || - typeof month === "undefined" || - Number.isNaN(month) || - typeof day === "undefined" || - Number.isNaN(day) - ) { - throw new Error("Invalid date"); - } - let date: Date; - if ( - typeof hour === "undefined" || - Number.isNaN(hour) || - typeof minute === "undefined" || - Number.isNaN(minute) || - typeof second === "undefined" || - Number.isNaN(second) - ) { - date = new Date(year, month - 1, day); - } else { - date = new Date(year, month - 1, day, hour, minute, second); - } - if (Number.isNaN(+date)) { - throw new Error("Invalid date"); - } - return date; - } catch (e) { - log.error(`Failed to parseEXIFDate ${dateTimeString}`, e); - return null; - } -} - -export function parseEXIFLocation( - gpsLatitude: number[], - gpsLatitudeRef: string, - gpsLongitude: number[], - gpsLongitudeRef: string, -) { - try { - if ( - !Array.isArray(gpsLatitude) || - !Array.isArray(gpsLongitude) || - gpsLatitude.length !== 3 || - gpsLongitude.length !== 3 - ) { - throw new Error("Invalid Exif location"); - } - const latitude = convertDMSToDD( - gpsLatitude[0], - gpsLatitude[1], - gpsLatitude[2], - gpsLatitudeRef, - ); - const longitude = convertDMSToDD( - gpsLongitude[0], - gpsLongitude[1], - gpsLongitude[2], - gpsLongitudeRef, - ); - return { latitude, longitude }; - } catch (e) { - const p = { - gpsLatitude, - gpsLatitudeRef, - gpsLongitude, - gpsLongitudeRef, - }; - log.error(`Failed to parse Exif location ${JSON.stringify(p)}`, e); - return { ...NULL_LOCATION }; - } -} - -function convertDMSToDD( - degrees: number, - minutes: number, - seconds: number, - direction: string, -) { - let dd = degrees + minutes / 60 + seconds / (60 * 60); - if (direction === "S" || direction === "W") dd *= -1; - return dd; -} - -export function getEXIFLocation(exifData: ParsedEXIFData): Location { - if (!exifData || (!exifData.latitude && exifData.latitude !== 0)) { - return { ...NULL_LOCATION }; - } - return { latitude: exifData.latitude, longitude: exifData.longitude }; -} - -export function getEXIFTime(exifData: ParsedEXIFData): number { - if (!exifData) { - return null; - } - const dateTime = - exifData.DateTimeOriginal ?? - exifData.DateCreated ?? - exifData.CreateDate ?? - exifData.MetadataDate ?? - exifData.ModifyDate; - if (!dateTime) { - return null; - } - return validateAndGetCreationUnixTimeInMicroSeconds(dateTime); -} diff --git a/web/yarn.lock b/web/yarn.lock index cb006b8edd..caf41595f2 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -2486,11 +2486,6 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== -exifr@^7.1.3: - version "7.1.3" - resolved "https://registry.yarnpkg.com/exifr/-/exifr-7.1.3.tgz#f6218012c36dbb7d843222011b27f065fddbab6f" - integrity sha512-g/aje2noHivrRSLbAUtBPWFbxKdKhgj/xr1vATDdUXPOFYJlQ62Ft0oy+72V6XLIpDJfHs6gXLbBLAolqOXYRw== - exifreader@^4: version "4.23.3" resolved "https://registry.yarnpkg.com/exifreader/-/exifreader-4.23.3.tgz#3389c2dab3ab2501562ebdef4115ea34ab9d9aa4" From 777f9e9704bddaf866043d0eb4316a9fd879dabd Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 15:42:24 +0530 Subject: [PATCH 064/211] - @xmldom/xmldom (In next commit we'll remove it from the indexer) --- web/apps/photos/package.json | 1 - web/docs/dependencies.md | 7 ++----- web/packages/new/photos/services/exif.ts | 12 +++++++++++ web/packages/new/photos/services/ml/worker.ts | 20 ------------------- 4 files changed, 14 insertions(+), 26 deletions(-) diff --git a/web/apps/photos/package.json b/web/apps/photos/package.json index f8b043adce..1a4f265b3e 100644 --- a/web/apps/photos/package.json +++ b/web/apps/photos/package.json @@ -10,7 +10,6 @@ "@ente/eslint-config": "*", "@ente/shared": "*", "@stripe/stripe-js": "^1.13.2", - "@xmldom/xmldom": "^0.8.10", "bip39": "^3.0.4", "bs58": "^5.0.0", "chrono-node": "^2.2.6", diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index 506df1c0ad..c28824b288 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -201,11 +201,8 @@ For more details, see [translations.md](translations.md). ## Media - [ExifReader](https://github.com/mattiasw/ExifReader) is used for Exif - parsing. We also need its optional peer dependency - [@xmldom/xmldom](https://github.com/xmldom/xmldom) since the browser's - DOMParser is not available in web workers. - [piexifjs](https://github.com/hMatoba/piexifjs) is used for writing back - Exif (only supports JPEG). + parsing. [piexifjs](https://github.com/hMatoba/piexifjs) is used for writing + back Exif (only supports JPEG). - [jszip](https://github.com/Stuk/jszip) is used for reading zip files in the web code (Live photos are zip files under the hood). Note that the desktop diff --git a/web/packages/new/photos/services/exif.ts b/web/packages/new/photos/services/exif.ts index 5e7a7ea8ec..544d367419 100644 --- a/web/packages/new/photos/services/exif.ts +++ b/web/packages/new/photos/services/exif.ts @@ -1,3 +1,4 @@ +import { inWorker } from "@/base/env"; import log from "@/base/log"; import { parseMetadataDate, @@ -477,6 +478,17 @@ export type RawExifTags = Omit & { * to know about ExifReader specifically. */ export const extractRawExif = async (blob: Blob): Promise => { + // The browser's DOMParser is not available in web workers. So if this + // function gets called in from a web worker, then it would not be able to + // parse XMP tags. + // + // There is a way around this problem, by also installing ExifReader's + // optional peer dependency "@xmldom/xmldom". But since we currently have no + // use case for calling this code in a web worker, we just abort immediately + // to let future us know that we need to install it. + if (inWorker()) + throw new Error("DOMParser is not available in web workers"); + const tags = await ExifReader.load(await blob.arrayBuffer(), { async: true, expanded: true, diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index 2fe408f9bb..2434b30779 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -8,10 +8,8 @@ import type { EnteFile } from "@/new/photos/types/file"; import { fileLogID } from "@/new/photos/utils/file"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; -import { DOMParser } from "@xmldom/xmldom"; import { expose, wrap } from "comlink"; import downloadManager from "../download"; -import { extractRawExif, type RawExifTags } from "../exif"; import { getAllLocalFiles, getLocalTrashedFiles } from "../files"; import type { UploadItem } from "../upload/types"; import { @@ -105,24 +103,6 @@ export class MLWorker { // Initialize the downloadManager running in the web worker with the // user's token. It'll be used to download files to index if needed. await downloadManager.init(await ensureAuthToken()); - - // Normally, DOMParser is available to web code, so our Exif library - // (ExifReader) has an optional dependency on the the non-browser - // alternative DOMParser provided by @xmldom/xmldom. - // - // But window.DOMParser is not available to web workers. - // - // So we need to get ExifReader to use the @xmldom/xmldom version. - // ExifReader references it using the following code: - // - // __non_webpack_require__('@xmldom/xmldom') - // - // So we need to explicitly reference it to ensure that it does not get - // tree shaken by webpack. But ensuring it is part of the bundle does - // not seem to work (for reasons I don't yet understand), so we also - // need to monkey patch it (This also ensures that it is not tree - // shaken). - globalThis.DOMParser = DOMParser; } /** From ca1039884f61bc21fe0d126a196acc840b3475d2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 15:50:10 +0530 Subject: [PATCH 065/211] Remove exif extraction during indexing --- web/packages/new/photos/services/ml/worker.ts | 88 ++++--------------- 1 file changed, 16 insertions(+), 72 deletions(-) diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index 2434b30779..ed64f4504d 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -351,19 +351,8 @@ const syncWithLocalFilesAndGetFilesToIndex = async ( /** * Index file, save the persist the results locally, and put them on remote. * - * [Note: ML indexing does more ML] - * - * Nominally, and primarily, indexing a file involves computing its various ML - * embeddings: faces and CLIP. However, since this is a occasion where we have - * the original file in memory, it is a great time to also compute other derived - * data related to the file (instead of re-downloading it again). - * - * So this function also does things that are not related to ML and/or indexing: - * - * - Extracting Exif. - * - Saving face crops. - * - * --- + * Indexing a file involves computing its various ML embeddings: faces and CLIP. + * Since we have the original file in memory, we also save the face crops. * * [Note: Transient and permanent indexing failures] * @@ -413,11 +402,6 @@ const index = async ( const existingRemoteFaceIndex = remoteDerivedData?.parsed?.face; const existingRemoteCLIPIndex = remoteDerivedData?.parsed?.clip; - // exif is expected to be a JSON object in the shape of RawExifTags, but - // this function don't care what's inside it and can just treat it as an - // opaque blob. - const existingExif = remoteDerivedData?.raw.exif; - let existingFaceIndex: FaceIndex | undefined; if ( existingRemoteFaceIndex && @@ -438,8 +422,8 @@ const index = async ( existingCLIPIndex = { embedding }; } - // See if we already have all the mandatory derived data fields. If so, just - // update our local db and return. + // If we already have all the derived data fields then just update our local + // db and return. if (existingFaceIndex && existingCLIPIndex) { try { @@ -456,8 +440,7 @@ const index = async ( // There is at least one derived data type that still needs to be indexed. - // Videos will not have an original blob whilst having a renderable blob. - const { originalImageBlob, renderableBlob } = await indexableBlobs( + const { renderableBlob } = await indexableBlobs( enteFile, uploadItem, electron, @@ -481,15 +464,13 @@ const index = async ( try { let faceIndex: FaceIndex; let clipIndex: CLIPIndex; - let exif: unknown; const startTime = Date.now(); try { - [faceIndex, clipIndex, exif] = await Promise.all([ + [faceIndex, clipIndex] = await Promise.all([ existingFaceIndex ?? indexFaces(enteFile, image, electron), existingCLIPIndex ?? indexCLIP(image, electron), - existingExif ?? tryExtractExif(originalImageBlob, f), ]); } catch (e) { // See: [Note: Transient and permanent indexing failures] @@ -503,7 +484,6 @@ const index = async ( const msg = []; if (!existingFaceIndex) msg.push(`${faceIndex.faces.length} faces`); if (!existingCLIPIndex) msg.push("clip"); - if (!existingExif && originalImageBlob) msg.push("exif"); return `Indexed ${msg.join(" and ")} in ${f} (${ms} ms)`; }); @@ -528,23 +508,17 @@ const index = async ( ...existingRawDerivedData, face: remoteFaceIndex, clip: remoteCLIPIndex, - ...(exif ? { exif } : {}), }; - if (existingFaceIndex && existingCLIPIndex && !exif) { - // If we were indexing just for exif, but exif generation didn't - // happen, there is no need to upload. - } else { - log.debug(() => ["Uploading derived data", rawDerivedData]); + log.debug(() => ["Uploading derived data", rawDerivedData]); - try { - await putDerivedData(enteFile, rawDerivedData); - } catch (e) { - // See: [Note: Transient and permanent indexing failures] - log.error(`Failed to put derived data for ${f}`, e); - if (isHTTP4xxError(e)) await markIndexingFailed(enteFile.id); - throw e; - } + try { + await putDerivedData(enteFile, rawDerivedData); + } catch (e) { + // See: [Note: Transient and permanent indexing failures] + log.error(`Failed to put derived data for ${f}`, e); + if (isHTTP4xxError(e)) await markIndexingFailed(enteFile.id); + throw e; } try { @@ -555,7 +529,8 @@ const index = async ( } catch (e) { // Not sure if DB failures should be considered permanent or // transient. There isn't a known case where writing to the local - // indexedDB would fail. + // indexedDB should systematically fail. It could fail if there was + // no space on device, but that's eminently retriable. log.error(`Failed to save indexes for ${f}`, e); throw e; } @@ -575,34 +550,3 @@ const index = async ( image.bitmap.close(); } }; - -/** - * A helper function that tries to extract the raw Exif, but returns `undefined` - * if something goes wrong (or it isn't possible) instead of throwing. - * - * Exif extraction is not a critical item, we don't want the actual indexing to - * fail because we were unable to extract Exif. This is not rare: one scenario - * is if we were trying to index a file in an exotic format. The ML indexing - * will succeed (because we convert it to a renderable blob), but the Exif - * extraction will fail (since it needs the original blob, but the original blob - * can be an arbitrary format). - * - * @param originalImageBlob A {@link Blob} containing the original data for the - * image (or the image component of a live photo) whose Exif we're trying to - * extract. If this is not available, we skip the extraction and return - * `undefined`. - * - * @param f The {@link fileLogID} for the file this blob corresponds to. - */ -export const tryExtractExif = async ( - originalImageBlob: Blob | undefined, - f: string, -): Promise => { - if (!originalImageBlob) return undefined; - try { - return await extractRawExif(originalImageBlob); - } catch (e) { - log.warn(`Ignoring error during Exif extraction for ${f}`, e); - return undefined; - } -}; From 6967d1235ed1b4bc3e940f31d8e503b8dc72a918 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 15:55:10 +0530 Subject: [PATCH 066/211] derived --- .../new/photos/services/ml/embedding.ts | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/web/packages/new/photos/services/ml/embedding.ts b/web/packages/new/photos/services/ml/embedding.ts index 2c96f43152..c0adb2cdb9 100644 --- a/web/packages/new/photos/services/ml/embedding.ts +++ b/web/packages/new/photos/services/ml/embedding.ts @@ -10,29 +10,26 @@ import { type RemoteCLIPIndex } from "./clip"; import { type RemoteFaceIndex } from "./face"; /** - * [Note: Derived embeddings and other metadata] + * [Note: Derived embeddings model] * - * The APIs they deal with derived data started in a ML context, and would store - * embeddings generated by particular models. Thus the API endpoints use the - * name "embedding", and are parameterized by a "model" enum. + * The API endpoints related to embeddings and are parameterized by a "model" + * enum. This is a bit of misnomer, since the contents of the payload are not + * just the raw embeddings themselves, but also additional data generated by the + * ML model. * - * Next step in the evolution was that instead of just storing the embedding, - * the code also started storing various additional data generated by the ML - * model. For example, the face indexing process generates multiple face - * embeddings per file, each with an associated detection box. So instead of - * storing just a singular embedding, the data that got stored was this entire - * face index structure containing multiple embeddings and associated data. + * For example, the face indexing process generates multiple face embeddings per + * file, each with an associated detection box. So instead of storing just a + * singular embedding, the data is an entire face index structure containing + * multiple embeddings and associated data. * * Further down, it was realized that the fan out caused on remote when trying - * to fetch all derived data - both ML ("clip", "face") and non-ML ("exif") - - * was problematic, and also their raw JSON was unnecessarily big. To deal with - * these better, we now have a single "derived" model type, whose data is a - * gzipped map of the form: + * to fetch both CLIP and face embeddings was problematic, and also that their + * raw JSON was unnecessarily big. To deal with these better, we now have a + * single "derived" model type, whose data is a gzipped map of the form: * * { * "face": ... the face indexing result ... * "clip": ... the CLIP indexing result ... - * "exif": ... the Exif extracted from the file ... * ... more in the future ... * } */ From 959f887d2f365523c9fbfc028bb249596106b600 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 16:08:20 +0530 Subject: [PATCH 067/211] Remove unused flexibility --- web/packages/new/photos/services/ml/blob.ts | 96 +++++-------------- web/packages/new/photos/services/ml/crop.ts | 4 +- web/packages/new/photos/services/ml/worker.ts | 8 +- 3 files changed, 31 insertions(+), 77 deletions(-) diff --git a/web/packages/new/photos/services/ml/blob.ts b/web/packages/new/photos/services/ml/blob.ts index d52772b6a4..6364d37667 100644 --- a/web/packages/new/photos/services/ml/blob.ts +++ b/web/packages/new/photos/services/ml/blob.ts @@ -10,41 +10,10 @@ import DownloadManager from "../download"; import type { UploadItem } from "../upload/types"; /** - * A pair of blobs - the original, and a possibly converted "renderable" one - - * for a file that we're trying to index. - */ -export interface IndexableBlobs { - /** - * The original file's data (as a {@link Blob}). - * - * - For images this is guaranteed to be present. - * - For videos it will not be present. - * - For live photos it will the (original) image component of the live - * photo. - */ - originalImageBlob: Blob | undefined; - /** - * The original (if the browser possibly supports rendering this type of - * images) or otherwise a converted JPEG blob. - * - * This blob is meant to be used to construct the {@link ImageBitmap} - * that'll be used for further operations that need access to the RGB data - * of the image. - * - * - For images this is constructed from the image. - * - For videos this is constructed from the thumbnail. - * - For live photos this is constructed from the image component of the - * live photo. - */ - renderableBlob: Blob; -} - -/** - * Indexable blobs augmented with the image bitmap and RGBA data. + * An image bitmap and its RGBA data. * * This is data structure containing data about an image in all formats that the - * various indexing steps need. Consolidating all the data here and parsing them - * in one go obviates the need for each indexing step to roll their own parsing. + * various indexing steps need. */ export interface ImageBitmapAndData { /** @@ -66,7 +35,7 @@ export interface ImageBitmapAndData { * Create an {@link ImageBitmap} from the given {@link imageBlob}, and return * both the image bitmap and its {@link ImageData}. */ -export const imageBitmapAndData = async ( +export const createImageBitmapAndData = async ( imageBlob: Blob, ): Promise => { const imageBitmap = await createImageBitmap(imageBlob); @@ -83,15 +52,15 @@ export const imageBitmapAndData = async ( }; /** - * Return a pair of blobs for the given data - the original, and a renderable - * one (possibly involving a JPEG conversion). + * Return a renderable blob (converting to JPEG if needed) for the given data. * * The blob from the relevant image component is either constructed using the * given {@link uploadItem} if present, otherwise it is downloaded from remote. * - * - For images the original is used. - * - For live photos the original image component is used. - * - For videos the thumbnail is used. + * - For images it is constructed from the image. + * - For videos it is constructed from the thumbnail. + * - For live photos it is constructed from the image component of the live + * photo. * * Then, if the image blob we have seems to be something that the browser cannot * handle, we convert it into a JPEG blob so that it can subsequently be used to @@ -107,34 +76,28 @@ export const imageBitmapAndData = async ( * witness that we're actually running in our desktop app (and thus can safely * call our Node.js layer for various functionality). */ -export const indexableBlobs = async ( +export const fetchRenderableBlob = async ( enteFile: EnteFile, uploadItem: UploadItem | undefined, electron: ElectronMLWorker, -): Promise => +): Promise => uploadItem - ? await indexableUploadItemBlobs(enteFile, uploadItem, electron) - : await indexableEnteFileBlobs(enteFile); + ? await fetchRenderableUploadItemBlob(enteFile, uploadItem, electron) + : await fetchRenderableEnteFileBlob(enteFile); -const indexableUploadItemBlobs = async ( +const fetchRenderableUploadItemBlob = async ( enteFile: EnteFile, uploadItem: UploadItem, electron: ElectronMLWorker, ) => { const fileType = enteFile.metadata.fileType; - let originalImageBlob: Blob | undefined; - let renderableBlob: Blob; if (fileType == FileType.video) { const thumbnailData = await DownloadManager.getThumbnail(enteFile); - renderableBlob = new Blob([ensure(thumbnailData)]); + return new Blob([ensure(thumbnailData)]); } else { - originalImageBlob = await readNonVideoUploadItem(uploadItem, electron); - renderableBlob = await renderableImageBlob( - enteFile.metadata.title, - originalImageBlob, - ); + const blob = await readNonVideoUploadItem(uploadItem, electron); + return renderableImageBlob(enteFile.metadata.title, blob); } - return { originalImageBlob, renderableBlob }; }; /** @@ -173,39 +136,32 @@ const readNonVideoUploadItem = async ( }; /** - * Return a pair of blobs for the given file - the original, and a renderable - * one (possibly involving a JPEG conversion). + * Return a renderable one (possibly involving a JPEG conversion) blob for the + * given {@link EnteFile}. * - * - The original will be downloaded if needed - * - The original will be converted to JPEG if needed + * - The original will be downloaded if needed. + * - The original will be converted to JPEG if needed. */ -export const indexableEnteFileBlobs = async ( +export const fetchRenderableEnteFileBlob = async ( enteFile: EnteFile, -): Promise => { +): Promise => { const fileType = enteFile.metadata.fileType; if (fileType == FileType.video) { const thumbnailData = await DownloadManager.getThumbnail(enteFile); - return { - originalImageBlob: undefined, - renderableBlob: new Blob([ensure(thumbnailData)]), - }; + return new Blob([ensure(thumbnailData)]); } const fileStream = await DownloadManager.getFile(enteFile); const originalImageBlob = await new Response(fileStream).blob(); - let renderableBlob: Blob; if (fileType == FileType.livePhoto) { const { imageFileName, imageData } = await decodeLivePhoto( enteFile.metadata.title, originalImageBlob, ); - renderableBlob = await renderableImageBlob( - imageFileName, - new Blob([imageData]), - ); + return renderableImageBlob(imageFileName, new Blob([imageData])); } else if (fileType == FileType.image) { - renderableBlob = await renderableImageBlob( + return await renderableImageBlob( enteFile.metadata.title, originalImageBlob, ); @@ -213,6 +169,4 @@ export const indexableEnteFileBlobs = async ( // A layer above us should've already filtered these out. throw new Error(`Cannot index unsupported file type ${fileType}`); } - - return { originalImageBlob, renderableBlob }; }; diff --git a/web/packages/new/photos/services/ml/crop.ts b/web/packages/new/photos/services/ml/crop.ts index d42d7d48fc..c12a74d5c0 100644 --- a/web/packages/new/photos/services/ml/crop.ts +++ b/web/packages/new/photos/services/ml/crop.ts @@ -1,7 +1,7 @@ import { blobCache } from "@/base/blob-cache"; import { ensure } from "@/utils/ensure"; import type { EnteFile } from "../../types/file"; -import { indexableEnteFileBlobs } from "./blob"; +import { fetchRenderableEnteFileBlob } from "./blob"; import { type Box, type FaceIndex } from "./face"; import { clamp } from "./math"; @@ -26,7 +26,7 @@ export const regenerateFaceCrops = async ( enteFile: EnteFile, faceIndex: FaceIndex, ) => { - const { renderableBlob } = await indexableEnteFileBlobs(enteFile); + const renderableBlob = await fetchRenderableEnteFileBlob(enteFile); const imageBitmap = await createImageBitmap(renderableBlob); try { diff --git a/web/packages/new/photos/services/ml/worker.ts b/web/packages/new/photos/services/ml/worker.ts index ed64f4504d..f41ae397d1 100644 --- a/web/packages/new/photos/services/ml/worker.ts +++ b/web/packages/new/photos/services/ml/worker.ts @@ -13,8 +13,8 @@ import downloadManager from "../download"; import { getAllLocalFiles, getLocalTrashedFiles } from "../files"; import type { UploadItem } from "../upload/types"; import { - imageBitmapAndData, - indexableBlobs, + createImageBitmapAndData, + fetchRenderableBlob, type ImageBitmapAndData, } from "./blob"; import { @@ -440,7 +440,7 @@ const index = async ( // There is at least one derived data type that still needs to be indexed. - const { renderableBlob } = await indexableBlobs( + const renderableBlob = await fetchRenderableBlob( enteFile, uploadItem, electron, @@ -448,7 +448,7 @@ const index = async ( let image: ImageBitmapAndData; try { - image = await imageBitmapAndData(renderableBlob); + image = await createImageBitmapAndData(renderableBlob); } catch (e) { // If we cannot get the raw image data for the file, then retrying again // won't help (if in the future we enhance the underlying code for From 1bb4940e14e504df6b76103162a4dbbd4316a0bc Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Wed, 7 Aug 2024 16:46:02 +0530 Subject: [PATCH 068/211] Handle deletion from inFlight replica --- server/ente/filedata/filedata.go | 1 + server/pkg/controller/filedata/delete.go | 7 +- server/pkg/repo/filedata/repository.go | 118 +++++++++++++++-------- 3 files changed, 83 insertions(+), 43 deletions(-) diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go index 807f2b5e6b..20ff30aea1 100644 --- a/server/ente/filedata/filedata.go +++ b/server/ente/filedata/filedata.go @@ -40,6 +40,7 @@ type Row struct { LatestBucket string ReplicatedBuckets []string DeleteFromBuckets []string + InflightReplicas []string PendingSync bool IsDeleted bool LastSyncTime int64 diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go index e9ada5a0f8..1673e7c0fc 100644 --- a/server/pkg/controller/filedata/delete.go +++ b/server/pkg/controller/filedata/delete.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/ente-io/museum/ente/filedata" "github.com/ente-io/museum/pkg/repo" + fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata" "github.com/ente-io/museum/pkg/utils/time" log "github.com/sirupsen/logrus" "strconv" @@ -70,7 +71,7 @@ func (c *Controller) deleteFileData(qItem repo.QueueItem) { return } } - dbErr := c.Repo.RemoveBucketFromDeletedBuckets(fileDataRow, bucketID) + dbErr := c.Repo.RemoveBucket(fileDataRow, bucketID, fileDataRepo.DeletionColumn) if dbErr != nil { ctxLogger.WithError(dbErr).Error("Failed to remove from db") return @@ -86,7 +87,7 @@ func (c *Controller) deleteFileData(qItem repo.QueueItem) { return } } - dbErr := c.Repo.RemoveBucketFromReplicatedBuckets(fileDataRow, bucketID) + dbErr := c.Repo.RemoveBucket(fileDataRow, bucketID, fileDataRepo.ReplicationColumn) if dbErr != nil { ctxLogger.WithError(dbErr).Error("Failed to remove from db") return @@ -100,7 +101,7 @@ func (c *Controller) deleteFileData(qItem repo.QueueItem) { return } } - dbErr := c.Repo.DeleteFileData(context.Background(), fileDataRow.FileID, fileDataRow.Type, fileDataRow.LatestBucket) + dbErr := c.Repo.DeleteFileData(context.Background(), fileDataRow) if dbErr != nil { ctxLogger.WithError(dbErr).Error("Failed to remove from db") return diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go index 8b56a76d33..75c409a6c6 100644 --- a/server/pkg/repo/filedata/repository.go +++ b/server/pkg/repo/filedata/repository.go @@ -3,6 +3,7 @@ package filedata import ( "context" "database/sql" + "fmt" "github.com/ente-io/museum/ente" "github.com/ente-io/museum/ente/filedata" "github.com/ente-io/stacktrace" @@ -14,6 +15,12 @@ type Repository struct { DB *sql.DB } +const ( + ReplicationColumn = "replicated_buckets" + DeletionColumn = "delete_from_buckets" + InflightRepColumn = "inflight_rep_buckets" +) + func (r *Repository) InsertOrUpdate(ctx context.Context, data filedata.Row) error { query := ` INSERT INTO file_data @@ -26,7 +33,7 @@ func (r *Repository) InsertOrUpdate(ctx context.Context, data filedata.Row) erro delete_from_buckets = array( SELECT DISTINCT elem FROM unnest( array_append( - array_cat(file_data.replicated_buckets, file_data.delete_from_buckets), + array_cat(array_cat(file_data.replicated_buckets, file_data.delete_from_buckets), file_data.inflight_rep_buckets), CASE WHEN file_data.latest_bucket != EXCLUDED.latest_bucket THEN file_data.latest_bucket END ) ) AS elem @@ -45,7 +52,7 @@ func (r *Repository) InsertOrUpdate(ctx context.Context, data filedata.Row) erro } func (r *Repository) GetFilesData(ctx context.Context, oType ente.ObjectType, fileIDs []int64) ([]filedata.Row, error) { - rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at + rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, inflight_rep_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at FROM file_data WHERE data_type = $1 AND file_id = ANY($2)`, string(oType), pq.Array(fileIDs)) if err != nil { @@ -55,7 +62,7 @@ func (r *Repository) GetFilesData(ctx context.Context, oType ente.ObjectType, fi } func (r *Repository) GetFileData(ctx context.Context, fileIDs int64) ([]filedata.Row, error) { - rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at + rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets,inflight_rep_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at FROM file_data WHERE file_id = $1`, fileIDs) if err != nil { @@ -64,65 +71,97 @@ func (r *Repository) GetFileData(ctx context.Context, fileIDs int64) ([]filedata return convertRowsToFilesData(rows) } -func (r *Repository) RemoveBucketFromDeletedBuckets(row filedata.Row, bucketID string) error { - query := ` - UPDATE file_data - SET delete_from_buckets = array( - SELECT DISTINCT elem FROM unnest( - array_remove( - file_data.delete_from_buckets, - $1 - ) - ) AS elem - WHERE elem IS NOT NULL - ) - WHERE file_id = $2 AND data_type = $3 and is_deleted = true` - result, err := r.DB.Exec(query, bucketID, row.FileID, string(row.Type)) +func (r *Repository) AddBucket(row filedata.Row, bucketID string, columnName string) error { + query := fmt.Sprintf(` + UPDATE file_data + SET %s = array( + SELECT DISTINCT elem FROM unnest( + array_append(file_data.%s, $1) + ) AS elem + ) + WHERE file_id = $2 AND data_type = $3 and user_id = $4`, columnName, columnName) + result, err := r.DB.Exec(query, bucketID, row.FileID, string(row.Type), row.UserID) if err != nil { - return stacktrace.Propagate(err, "failed to remove bucket from deleted buckets") + return stacktrace.Propagate(err, "failed to add bucket to "+columnName) } rowsAffected, err := result.RowsAffected() if err != nil { return stacktrace.Propagate(err, "") } if rowsAffected == 0 { - return stacktrace.NewError("bucket not removed from deleted buckets") + return stacktrace.NewError("bucket not added to " + columnName) } return nil } -func (r *Repository) RemoveBucketFromReplicatedBuckets(row filedata.Row, bucketID string) error { - query := ` - UPDATE file_data - SET replicated_buckets = array( - SELECT DISTINCT elem FROM unnest( - array_remove( - file_data.replicated_buckets, - $1 - ) - ) AS elem - WHERE elem IS NOT NULL - ) - WHERE file_id = $2 AND data_type = $3` - result, err := r.DB.Exec(query, bucketID, row.FileID, string(row.Type)) +func (r *Repository) RemoveBucket(row filedata.Row, bucketID string, columnName string) error { + query := fmt.Sprintf(` + UPDATE file_data + SET %s = array( + SELECT DISTINCT elem FROM unnest( + array_remove( + file_data.%s, + $1 + ) + ) AS elem + WHERE elem IS NOT NULL + ) + WHERE file_id = $2 AND data_type = $3 and user_id = $4`, columnName, columnName) + result, err := r.DB.Exec(query, bucketID, row.FileID, string(row.Type), row.UserID) + if err != nil { + return stacktrace.Propagate(err, "failed to remove bucket from "+columnName) + } + rowsAffected, err := result.RowsAffected() + if err != nil { + return stacktrace.Propagate(err, "") + } + if rowsAffected == 0 { + return stacktrace.NewError("bucket not removed from " + columnName) + } + return nil +} + +func (r *Repository) MoveBetweenBuckets(row filedata.Row, bucketID string, sourceColumn string, destColumn string) error { + query := fmt.Sprintf(` + UPDATE file_data + SET %s = array( + SELECT DISTINCT elem FROM unnest( + array_append( + file_data.%s, + $1 + ) + ) AS elem + WHERE elem IS NOT NULL + ), + %s = array( + SELECT DISTINCT elem FROM unnest( + array_remove( + file_data.%s, + $1 + ) + ) AS elem + WHERE elem IS NOT NULL + ) + WHERE file_id = $2 AND data_type = $3 and user_id = $4`, destColumn, destColumn, sourceColumn, sourceColumn) + result, err := r.DB.Exec(query, bucketID, row.FileID, string(row.Type), row.UserID) if err != nil { - return stacktrace.Propagate(err, "failed to remove bucket from replicated buckets") + return stacktrace.Propagate(err, "failed to move bucket from "+sourceColumn+" to "+destColumn) } rowsAffected, err := result.RowsAffected() if err != nil { return stacktrace.Propagate(err, "") } if rowsAffected == 0 { - return stacktrace.NewError("bucket not removed from deleted buckets") + return stacktrace.NewError("bucket not moved from " + sourceColumn + " to " + destColumn) } return nil } -func (r *Repository) DeleteFileData(ctx context.Context, fileID int64, oType ente.ObjectType, latestBucketID string) error { +func (r *Repository) DeleteFileData(ctx context.Context, row filedata.Row) error { query := ` DELETE FROM file_data -WHERE file_id = $1 AND data_type = $2 AND latest_bucket = $3 AND replicated_buckets = ARRAY[]::s3region[] AND delete_from_buckets = ARRAY[]::s3region[]` - res, err := r.DB.ExecContext(ctx, query, fileID, string(oType), latestBucketID) +WHERE file_id = $1 AND data_type = $2 AND latest_bucket = $3 AND user_id = $4 AND replicated_buckets = ARRAY[]::s3region[] AND delete_from_buckets = ARRAY[]::s3region[]` + res, err := r.DB.ExecContext(ctx, query, row.FileID, string(row.Type), row.LatestBucket, row.UserID) if err != nil { return stacktrace.Propagate(err, "") } @@ -141,12 +180,11 @@ func convertRowsToFilesData(rows *sql.Rows) ([]filedata.Row, error) { var filesData []filedata.Row for rows.Next() { var fileData filedata.Row - err := rows.Scan(&fileData.FileID, &fileData.UserID, &fileData.Type, &fileData.Size, &fileData.LatestBucket, pq.Array(&fileData.ReplicatedBuckets), pq.Array(&fileData.DeleteFromBuckets), &fileData.PendingSync, &fileData.IsDeleted, &fileData.LastSyncTime, &fileData.CreatedAt, &fileData.UpdatedAt) + err := rows.Scan(&fileData.FileID, &fileData.UserID, &fileData.Type, &fileData.Size, &fileData.LatestBucket, pq.Array(&fileData.ReplicatedBuckets), pq.Array(&fileData.DeleteFromBuckets), pq.Array(&fileData.InflightReplicas), &fileData.PendingSync, &fileData.IsDeleted, &fileData.LastSyncTime, &fileData.CreatedAt, &fileData.UpdatedAt) if err != nil { return nil, stacktrace.Propagate(err, "") } filesData = append(filesData, fileData) } return filesData, nil - } From fe97828328304757e7dac6685604967c0ea130a4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Wed, 7 Aug 2024 19:50:00 +0530 Subject: [PATCH 069/211] Remove debug log --- web/packages/new/photos/services/exif.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/web/packages/new/photos/services/exif.ts b/web/packages/new/photos/services/exif.ts index 544d367419..882da37a53 100644 --- a/web/packages/new/photos/services/exif.ts +++ b/web/packages/new/photos/services/exif.ts @@ -1,5 +1,4 @@ import { inWorker } from "@/base/env"; -import log from "@/base/log"; import { parseMetadataDate, type ParsedMetadata, @@ -111,8 +110,6 @@ const parseDates = (tags: RawExifTags) => { const iptc = parseIPTCDates(tags); const xmp = parseXMPDates(tags); - log.debug(() => ["exif/dates", { exif, iptc, xmp }]); - return { DateTimeOriginal: valid(xmp.DateTimeOriginal) ?? From b7bd8c83bae2e32039d97fe44085fb018bfb13ff Mon Sep 17 00:00:00 2001 From: Aman Raj Singh Mourya Date: Thu, 8 Aug 2024 00:44:27 +0530 Subject: [PATCH 070/211] [mob][photos] Swipe lock for multiple files --- mobile/lib/events/file_swipe_lock_event.dart | 6 ++-- .../file_selection_actions_widget.dart | 28 +++++++++++++++++-- mobile/lib/ui/viewer/file/detail_page.dart | 21 ++++++++------ mobile/lib/ui/viewer/file/file_app_bar.dart | 20 ++++++------- .../lib/ui/viewer/file/file_bottom_bar.dart | 8 +++--- mobile/lib/ui/viewer/file/video_widget.dart | 6 ++-- .../lib/ui/viewer/file/video_widget_new.dart | 6 ++-- mobile/lib/ui/viewer/file/zoomable_image.dart | 6 ++-- 8 files changed, 65 insertions(+), 36 deletions(-) diff --git a/mobile/lib/events/file_swipe_lock_event.dart b/mobile/lib/events/file_swipe_lock_event.dart index 7e1a430202..48aed9bcf5 100644 --- a/mobile/lib/events/file_swipe_lock_event.dart +++ b/mobile/lib/events/file_swipe_lock_event.dart @@ -1,7 +1,7 @@ import "package:photos/events/event.dart"; class FileSwipeLockEvent extends Event { - final bool shouldSwipeLock; - - FileSwipeLockEvent(this.shouldSwipeLock); + final bool isGuestView; + final bool swipeLocked; + FileSwipeLockEvent(this.isGuestView, this.swipeLocked); } diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index 9485326abb..b37b17f671 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -8,6 +8,7 @@ import "package:logging/logging.dart"; import "package:modal_bottom_sheet/modal_bottom_sheet.dart"; import 'package:photos/core/configuration.dart'; import "package:photos/core/event_bus.dart"; +import "package:photos/events/file_swipe_lock_event.dart"; import "package:photos/events/people_changed_event.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/l10n.dart"; @@ -32,9 +33,9 @@ import 'package:photos/ui/components/action_sheet_widget.dart'; import "package:photos/ui/components/bottom_action_bar/selection_action_button_widget.dart"; import 'package:photos/ui/components/buttons/button_widget.dart'; import 'package:photos/ui/components/models/button_type.dart'; -// import 'package:photos/ui/sharing/manage_links_widget.dart'; import "package:photos/ui/sharing/show_images_prevew.dart"; import "package:photos/ui/tools/collage/collage_creator_page.dart"; +import "package:photos/ui/viewer/file/detail_page.dart"; import "package:photos/ui/viewer/location/update_location_data_widget.dart"; import 'package:photos/utils/delete_file_util.dart'; import "package:photos/utils/dialog_util.dart"; @@ -271,7 +272,13 @@ class _FileSelectionActionsWidgetState ), ); } - + items.add( + SelectionActionButton( + icon: Icons.lock, + labelText: "Guest view", + onTap: _onGuestViewClick, + ), + ); items.add( SelectionActionButton( icon: Icons.grid_view_outlined, @@ -559,6 +566,23 @@ class _FileSelectionActionsWidgetState } } + Future _onGuestViewClick() async { + final List selectedFiles = widget.selectedFiles.files.toList(); + final page = DetailPage( + DetailPageConfiguration( + selectedFiles, + null, + 0, + "guest_view", + ), + ); + routeToPage(context, page, forceCustomPageRoute: true).ignore(); + WidgetsBinding.instance.addPostFrameCallback((_) { + Bus.instance.fire(FileSwipeLockEvent(true, false)); + }); + widget.selectedFiles.clearAll(); + } + Future _onArchiveClick() async { await changeVisibility( context, diff --git a/mobile/lib/ui/viewer/file/detail_page.dart b/mobile/lib/ui/viewer/file/detail_page.dart index 521a12ea69..bcf3d0540c 100644 --- a/mobile/lib/ui/viewer/file/detail_page.dart +++ b/mobile/lib/ui/viewer/file/detail_page.dart @@ -86,7 +86,8 @@ class _DetailPageState extends State { bool _hasLoadedTillEnd = false; final _enableFullScreenNotifier = ValueNotifier(false); bool _isFirstOpened = true; - bool isFileSwipeLocked = false; + bool isGuestView = false; + bool swipeLocked = false; late final StreamSubscription _fileSwipeLockEventSubscription; @@ -102,7 +103,8 @@ class _DetailPageState extends State { _fileSwipeLockEventSubscription = Bus.instance.on().listen((event) { setState(() { - isFileSwipeLocked = event.shouldSwipeLock; + isGuestView = event.isGuestView; + swipeLocked = event.swipeLocked; }); }); } @@ -138,12 +140,12 @@ class _DetailPageState extends State { " files .", ); return PopScope( - canPop: !isFileSwipeLocked, + canPop: !isGuestView, onPopInvoked: (didPop) async { - if (isFileSwipeLocked) { + if (isGuestView) { final authenticated = await _requestAuthentication(); if (authenticated) { - Bus.instance.fire(FileSwipeLockEvent(false)); + Bus.instance.fire(FileSwipeLockEvent(false, false)); } } }, @@ -176,7 +178,7 @@ class _DetailPageState extends State { _files![selectedIndex], _onEditFileRequested, widget.config.mode == DetailPageMode.minimalistic && - !isFileSwipeLocked, + !isGuestView, onFileRemoved: _onFileRemoved, userID: Configuration.instance.getUserID(), enableFullScreenNotifier: _enableFullScreenNotifier, @@ -235,10 +237,13 @@ class _DetailPageState extends State { } else { _selectedIndexNotifier.value = index; } + Bus.instance.fire(FileSwipeLockEvent(isGuestView, swipeLocked)); _preloadEntries(); }, - physics: _shouldDisableScroll || isFileSwipeLocked - ? const NeverScrollableScrollPhysics() + physics: _shouldDisableScroll || isGuestView + ? swipeLocked + ? const NeverScrollableScrollPhysics() + : const FastScrollPhysics(speedFactor: 4.0) : const FastScrollPhysics(speedFactor: 4.0), controller: _pageController, itemCount: _files!.length, diff --git a/mobile/lib/ui/viewer/file/file_app_bar.dart b/mobile/lib/ui/viewer/file/file_app_bar.dart index e67795888f..f6d83169c6 100644 --- a/mobile/lib/ui/viewer/file/file_app_bar.dart +++ b/mobile/lib/ui/viewer/file/file_app_bar.dart @@ -53,7 +53,7 @@ class FileAppBarState extends State { final List _actions = []; late final StreamSubscription _fileSwipeLockEventSubscription; - bool _isFileSwipeLocked = false; + bool isGuestView = false; @override void didUpdateWidget(FileAppBar oldWidget) { @@ -69,7 +69,7 @@ class FileAppBarState extends State { _fileSwipeLockEventSubscription = Bus.instance.on().listen((event) { setState(() { - _isFileSwipeLocked = event.shouldSwipeLock; + isGuestView = event.isGuestView; }); }); } @@ -124,19 +124,19 @@ class FileAppBarState extends State { switchOutCurve: Curves.easeInOut, child: AppBar( clipBehavior: Clip.none, - key: ValueKey(_isFileSwipeLocked), + key: ValueKey(isGuestView), iconTheme: const IconThemeData( color: Colors.white, ), //same for both themes leading: IconButton( icon: const Icon(Icons.arrow_back), onPressed: () { - _isFileSwipeLocked + isGuestView ? _requestAuthentication() : Navigator.of(context).pop(); }, ), - actions: shouldShowActions && !_isFileSwipeLocked ? _actions : [], + actions: shouldShowActions && !isGuestView ? _actions : [], elevation: 0, backgroundColor: const Color(0x00000000), ), @@ -306,7 +306,7 @@ class FileAppBarState extends State { const Padding( padding: EdgeInsets.all(8), ), - const Text("Swipe lock"), + const Text("Guest view"), ], ), ), @@ -329,7 +329,7 @@ class FileAppBarState extends State { } else if (value == 5) { await _handleUnHideRequest(context); } else if (value == 6) { - await _onSwipeLock(); + await _onTapGuestView(); } }, ), @@ -413,9 +413,9 @@ class FileAppBarState extends State { } } - Future _onSwipeLock() async { + Future _onTapGuestView() async { if (await LocalAuthentication().isDeviceSupported()) { - Bus.instance.fire(FileSwipeLockEvent(!_isFileSwipeLocked)); + Bus.instance.fire(FileSwipeLockEvent(!isGuestView, true)); } else { await showErrorDialog( context, @@ -432,7 +432,7 @@ class FileAppBarState extends State { "Please authenticate to view more photos and videos.", ); if (hasAuthenticated) { - Bus.instance.fire(FileSwipeLockEvent(false)); + Bus.instance.fire(FileSwipeLockEvent(false, false)); } } } diff --git a/mobile/lib/ui/viewer/file/file_bottom_bar.dart b/mobile/lib/ui/viewer/file/file_bottom_bar.dart index 9db7275918..a569656b29 100644 --- a/mobile/lib/ui/viewer/file/file_bottom_bar.dart +++ b/mobile/lib/ui/viewer/file/file_bottom_bar.dart @@ -47,7 +47,7 @@ class FileBottomBar extends StatefulWidget { class FileBottomBarState extends State { final GlobalKey shareButtonKey = GlobalKey(); - bool _isFileSwipeLocked = false; + bool isGuestView = false; late final StreamSubscription _fileSwipeLockEventSubscription; bool isPanorama = false; @@ -59,7 +59,7 @@ class FileBottomBarState extends State { _fileSwipeLockEventSubscription = Bus.instance.on().listen((event) { setState(() { - _isFileSwipeLocked = event.shouldSwipeLock; + isGuestView = event.isGuestView; }); }); } @@ -220,9 +220,9 @@ class FileBottomBarState extends State { valueListenable: widget.enableFullScreenNotifier, builder: (BuildContext context, bool isFullScreen, _) { return IgnorePointer( - ignoring: isFullScreen || _isFileSwipeLocked, + ignoring: isFullScreen || isGuestView, child: AnimatedOpacity( - opacity: isFullScreen || _isFileSwipeLocked ? 0 : 1, + opacity: isFullScreen || isGuestView ? 0 : 1, duration: const Duration(milliseconds: 250), curve: Curves.easeInOut, child: Align( diff --git a/mobile/lib/ui/viewer/file/video_widget.dart b/mobile/lib/ui/viewer/file/video_widget.dart index 2cbb14fed0..b9d06ecaff 100644 --- a/mobile/lib/ui/viewer/file/video_widget.dart +++ b/mobile/lib/ui/viewer/file/video_widget.dart @@ -48,7 +48,7 @@ class _VideoWidgetState extends State { final _progressNotifier = ValueNotifier(null); bool _isPlaying = false; final EnteWakeLock _wakeLock = EnteWakeLock(); - bool _isFileSwipeLocked = false; + bool isGuestView = false; late final StreamSubscription _fileSwipeLockEventSubscription; @@ -83,7 +83,7 @@ class _VideoWidgetState extends State { _fileSwipeLockEventSubscription = Bus.instance.on().listen((event) { setState(() { - _isFileSwipeLocked = event.shouldSwipeLock; + isGuestView = event.isGuestView; }); }); } @@ -188,7 +188,7 @@ class _VideoWidgetState extends State { ? _getVideoPlayer() : _getLoadingWidget(); final contentWithDetector = GestureDetector( - onVerticalDragUpdate: _isFileSwipeLocked + onVerticalDragUpdate: isGuestView ? null : (d) => { if (d.delta.dy > dragSensitivity) diff --git a/mobile/lib/ui/viewer/file/video_widget_new.dart b/mobile/lib/ui/viewer/file/video_widget_new.dart index dec11b3ca1..f1a91b77ff 100644 --- a/mobile/lib/ui/viewer/file/video_widget_new.dart +++ b/mobile/lib/ui/viewer/file/video_widget_new.dart @@ -47,7 +47,7 @@ class _VideoWidgetNewState extends State late StreamSubscription playingStreamSubscription; bool _isAppInFG = true; late StreamSubscription pauseVideoSubscription; - bool _isFileSwipeLocked = false; + bool isGuestView = false; late final StreamSubscription _fileSwipeLockEventSubscription; @@ -97,7 +97,7 @@ class _VideoWidgetNewState extends State _fileSwipeLockEventSubscription = Bus.instance.on().listen((event) { setState(() { - _isFileSwipeLocked = event.shouldSwipeLock; + isGuestView = event.isGuestView; }); }); } @@ -159,7 +159,7 @@ class _VideoWidgetNewState extends State ), fullscreen: const MaterialVideoControlsThemeData(), child: GestureDetector( - onVerticalDragUpdate: _isFileSwipeLocked + onVerticalDragUpdate: isGuestView ? null : (d) => { if (d.delta.dy > dragSensitivity) diff --git a/mobile/lib/ui/viewer/file/zoomable_image.dart b/mobile/lib/ui/viewer/file/zoomable_image.dart index 4df004b1a2..64a7b2e0e5 100644 --- a/mobile/lib/ui/viewer/file/zoomable_image.dart +++ b/mobile/lib/ui/viewer/file/zoomable_image.dart @@ -54,7 +54,7 @@ class _ZoomableImageState extends State { bool _isZooming = false; PhotoViewController _photoViewController = PhotoViewController(); final _scaleStateController = PhotoViewScaleStateController(); - bool _isFileSwipeLocked = false; + bool isGuestView = false; late final StreamSubscription _fileSwipeLockEventSubscription; @@ -75,7 +75,7 @@ class _ZoomableImageState extends State { _fileSwipeLockEventSubscription = Bus.instance.on().listen((event) { setState(() { - _isFileSwipeLocked = event.shouldSwipeLock; + isGuestView = event.isGuestView; }); }); } @@ -159,7 +159,7 @@ class _ZoomableImageState extends State { } final GestureDragUpdateCallback? verticalDragCallback = - _isZooming || _isFileSwipeLocked + _isZooming || isGuestView ? null : (d) => { if (!_isZooming) From 7cafa9ccb686436dc4f62a006ce7fb253ce119c4 Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Thu, 8 Aug 2024 04:00:59 +0530 Subject: [PATCH 071/211] fix(mob/panorama): add background pattern --- .../lib/ui/viewer/file/file_bottom_bar.dart | 7 +- .../viewer/file/panorama_viewer_screen.dart | 5 ++ mobile/pubspec.lock | 71 +++++++++++-------- mobile/pubspec.yaml | 9 ++- 4 files changed, 57 insertions(+), 35 deletions(-) diff --git a/mobile/lib/ui/viewer/file/file_bottom_bar.dart b/mobile/lib/ui/viewer/file/file_bottom_bar.dart index 9d9b00ebb6..9b60748386 100644 --- a/mobile/lib/ui/viewer/file/file_bottom_bar.dart +++ b/mobile/lib/ui/viewer/file/file_bottom_bar.dart @@ -22,6 +22,7 @@ import 'package:photos/utils/delete_file_util.dart'; import "package:photos/utils/file_util.dart"; import "package:photos/utils/panorama_util.dart"; import 'package:photos/utils/share_util.dart'; +import "package:photos/utils/thumbnail_util.dart"; class FileBottomBar extends StatefulWidget { final EnteFile file; @@ -296,10 +297,14 @@ class FileBottomBarState extends State { if (fetchedFile == null) { return; } + final fetchedThumbnail = await getThumbnail(file); Navigator.of(context).push( MaterialPageRoute( builder: (_) { - return PanoramaViewerScreen(file: fetchedFile); + return PanoramaViewerScreen( + file: fetchedFile, + thumbnail: fetchedThumbnail, + ); }, ), ).ignore(); diff --git a/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart b/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart index 4b05bbd6f9..b54b63879c 100644 --- a/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart +++ b/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart @@ -11,9 +11,11 @@ class PanoramaViewerScreen extends StatefulWidget { const PanoramaViewerScreen({ super.key, required this.file, + required this.thumbnail, }); final File file; + final Uint8List? thumbnail; @override State createState() => _PanoramaViewerScreenState(); @@ -139,6 +141,9 @@ class _PanoramaViewerScreenState extends State { croppedFullWidth: width, croppedFullHeight: height, sensorControl: control, + background: widget.thumbnail != null + ? Image.memory(widget.thumbnail!) + : null, child: Image.file( widget.file, ), diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index ca98122697..e9cec130c2 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -69,10 +69,10 @@ packages: dependency: "direct main" description: name: archive - sha256: "22600aa1e926be775fa5fe7e6894e7fb3df9efda8891c73f70fb3262399a432d" + sha256: cb6a278ef2dbb298455e1a713bda08524a175630ec643a242c399c932a0a1f7d url: "https://pub.dev" source: hosted - version: "3.4.10" + version: "3.6.1" args: dependency: transitive description: @@ -113,6 +113,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.6" + blurhash_ffi: + dependency: transitive + description: + name: blurhash_ffi + sha256: "941868602bb3bc34b0a7d630e4bf0e88e4c93af36090fe1cc0d63021c9a46cb3" + url: "https://pub.dev" + source: hosted + version: "1.2.6" boolean_selector: dependency: transitive description: @@ -799,10 +807,10 @@ packages: dependency: "direct main" description: name: flutter_local_notifications - sha256: f9a05409385b77b06c18f200a41c7c2711ebf7415669350bb0f8474c07bd40d1 + sha256: dd6676d8c2926537eccdf9f72128bbb2a9d0814689527b17f92c248ff192eaf3 url: "https://pub.dev" source: hosted - version: "17.0.0" + version: "17.2.1+2" flutter_local_notifications_linux: dependency: transitive description: @@ -815,10 +823,10 @@ packages: dependency: transitive description: name: flutter_local_notifications_platform_interface - sha256: "7cf643d6d5022f3baed0be777b0662cce5919c0a7b86e700299f22dc4ae660ef" + sha256: "85f8d07fe708c1bdcf45037f2c0109753b26ae077e9d9e899d55971711a4ea66" url: "https://pub.dev" source: hosted - version: "7.0.0+1" + version: "7.2.0" flutter_localizations: dependency: "direct main" description: flutter @@ -1304,18 +1312,18 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "7f0df31977cb2c0b88585095d168e689669a2cc9b97c309665e3386f3e9d341a" + sha256: "3f87a60e8c63aecc975dda1ceedbc8f24de75f09e4856ea27daf8958f2f0ce05" url: "https://pub.dev" source: hosted - version: "10.0.4" + version: "10.0.5" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: "06e98f569d004c1315b991ded39924b21af84cf14cc94791b8aea337d25b57f8" + sha256: "932549fb305594d82d7183ecd9fa93463e9914e1b67cacc34bc40906594a1806" url: "https://pub.dev" source: hosted - version: "3.0.3" + version: "3.0.5" leak_tracker_testing: dependency: transitive description: @@ -1440,10 +1448,10 @@ packages: dependency: transitive description: name: material_color_utilities - sha256: "0e0a020085b65b6083975e499759762399b4475f766c21668c4ecca34ea74e5a" + sha256: f7142bb1154231d7ea5f96bc7bde4bda2a0945d2806bb11670e30b850d56bdec url: "https://pub.dev" source: hosted - version: "0.8.0" + version: "0.11.1" media_extension: dependency: "direct main" description: @@ -1528,10 +1536,10 @@ packages: dependency: transitive description: name: meta - sha256: "7687075e408b093f36e6bbf6c91878cc0d4cd10f409506f7bc996f68220b9136" + sha256: bdb68674043280c3428e9ec998512fb681678676b3c54e773629ffe74419f8c7 url: "https://pub.dev" source: hosted - version: "1.12.0" + version: "1.15.0" mgrs_dart: dependency: transitive description: @@ -1682,10 +1690,11 @@ packages: panorama_viewer: dependency: "direct main" description: - name: panorama_viewer - sha256: "66b51a28921df33463968ebb411394551f6dbb065cc98474401820d6c2676e57" - url: "https://pub.dev" - source: hosted + path: "." + ref: HEAD + resolved-ref: "26ad55b2aa29dde14d640d6f1e17ce132a87910a" + url: "https://github.com/prateekmedia/panorama_viewer.git" + source: git version: "1.0.5" password_strength: dependency: "direct main" @@ -1867,10 +1876,10 @@ packages: dependency: transitive description: name: platform - sha256: "12220bb4b65720483f8fa9450b4332347737cf8213dd2840d8b2c823e47243ec" + sha256: "9b71283fc13df574056616011fb138fd3b793ea47cc509c189a6c3fa5f8a1a65" url: "https://pub.dev" source: hosted - version: "3.1.4" + version: "3.1.5" plugin_platform_interface: dependency: transitive description: @@ -2376,26 +2385,26 @@ packages: dependency: "direct dev" description: name: test - sha256: "7ee446762c2c50b3bd4ea96fe13ffac69919352bd3b4b17bac3f3465edc58073" + sha256: "7ee44229615f8f642b68120165ae4c2a75fe77ae2065b1e55ae4711f6cf0899e" url: "https://pub.dev" source: hosted - version: "1.25.2" + version: "1.25.7" test_api: dependency: transitive description: name: test_api - sha256: "9955ae474176f7ac8ee4e989dadfb411a58c30415bcfb648fa04b2b8a03afa7f" + sha256: "5b8a98dafc4d5c4c9c72d8b31ab2b23fc13422348d2997120294d3bac86b4ddb" url: "https://pub.dev" source: hosted - version: "0.7.0" + version: "0.7.2" test_core: dependency: transitive description: name: test_core - sha256: "2bc4b4ecddd75309300d8096f781c0e3280ca1ef85beda558d33fcbedc2eead4" + sha256: "55ea5a652e38a1dfb32943a7973f3681a60f872f8c3a05a14664ad54ef9c6696" url: "https://pub.dev" source: hosted - version: "0.6.0" + version: "0.6.4" timezone: dependency: transitive description: @@ -2674,10 +2683,10 @@ packages: dependency: transitive description: name: vm_service - sha256: "3923c89304b715fb1eb6423f017651664a03bf5f4b29983627c4da791f74a4ec" + sha256: f652077d0bdf60abe4c1f6377448e8655008eef28f128bc023f7b5e8dfeb48fc url: "https://pub.dev" source: hosted - version: "14.2.1" + version: "14.2.4" volume_controller: dependency: transitive description: @@ -2767,13 +2776,13 @@ packages: source: hosted version: "0.0.2" win32: - dependency: transitive + dependency: "direct overridden" description: name: win32 - sha256: "464f5674532865248444b4c3daca12bd9bf2d7c47f759ce2617986e7229494a8" + sha256: "015002c060f1ae9f41a818f2d5640389cc05283e368be19dc8d77cecb43c40c9" url: "https://pub.dev" source: hosted - version: "5.2.0" + version: "5.5.3" win32_registry: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index a03ba4143a..e090092713 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -22,7 +22,7 @@ dependencies: adaptive_theme: ^3.1.0 animate_do: ^2.0.0 animated_list_plus: ^0.5.2 - archive: ^3.1.2 + archive: ^3.6.1 background_fetch: ^1.2.1 battery_info: ^1.1.1 bip39: ^1.0.6 @@ -81,7 +81,7 @@ dependencies: flutter_image_compress: ^1.1.0 flutter_inappwebview: ^5.8.0 flutter_launcher_icons: ^0.13.1 - flutter_local_notifications: ^17.0.0 + flutter_local_notifications: ^17.2.1+2 flutter_localizations: sdk: flutter flutter_map: ^6.2.0 @@ -131,7 +131,9 @@ dependencies: open_mail_app: ^0.4.5 package_info_plus: ^4.1.0 page_transition: ^2.0.2 - panorama_viewer: ^1.0.5 + panorama_viewer: + git: + url: https://github.com/ente-io/panorama_viewer.git password_strength: ^0.2.0 path: #dart path_provider: ^2.1.1 @@ -195,6 +197,7 @@ dependency_overrides: ref: android_video_roation_fix path: packages/video_player/video_player/ watcher: ^1.1.0 + win32: ^5.5.3 flutter_intl: enabled: true From 67d0fb1c3192c780706a8d7dc61f5baf0eef1933 Mon Sep 17 00:00:00 2001 From: Guspan Tanadi <36249910+guspan-tanadi@users.noreply.github.com> Date: Thu, 8 Aug 2024 07:32:39 +0700 Subject: [PATCH 072/211] docs: section links Method --- docs/docs/auth/migration-guides/authy/index.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/auth/migration-guides/authy/index.md b/docs/docs/auth/migration-guides/authy/index.md index 1a92285472..c72becd001 100644 --- a/docs/docs/auth/migration-guides/authy/index.md +++ b/docs/docs/auth/migration-guides/authy/index.md @@ -101,7 +101,7 @@ to Ente Authenticator! > the codes yet, ignore this section. > > If the export itself failed, try using -> [**method 1**](#method-1-use-neerajs-export-tool) instead. +> [**method 1**](#method-1-use-neeraj-s-export-tool) instead. Usually, you should be able to import Bitwarden exports directly into Ente Authenticator. In case this didn't work for whatever reason, I've written a @@ -170,7 +170,7 @@ depending on which method you used to export your codes. 5. Select the JSON file that was made earlier. If this didn't work, refer to -[**method 2.1**](#method-21-if-the-export-worked-but-the-import-didnt).

+[**method 2.1**](#method-2-1-if-the-export-worked-but-the-import-didn-t).

And that's it! You have now successfully migrated from Authy to Ente Authenticator. From 6fbc8072255d0cc9fe86e07ad0fe083798b5c5a9 Mon Sep 17 00:00:00 2001 From: httpjamesm <51917118+httpjamesm@users.noreply.github.com> Date: Thu, 8 Aug 2024 00:09:55 -0400 Subject: [PATCH 073/211] feat: shakepay icon --- auth/assets/custom-icons/icons/shakepay.svg | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 auth/assets/custom-icons/icons/shakepay.svg diff --git a/auth/assets/custom-icons/icons/shakepay.svg b/auth/assets/custom-icons/icons/shakepay.svg new file mode 100644 index 0000000000..ea99ff4e82 --- /dev/null +++ b/auth/assets/custom-icons/icons/shakepay.svg @@ -0,0 +1,31 @@ + + + + + + + + + From f3b13042cae82720dfcc3b53f42103a0ad55ebcd Mon Sep 17 00:00:00 2001 From: httpjamesm <51917118+httpjamesm@users.noreply.github.com> Date: Thu, 8 Aug 2024 00:11:42 -0400 Subject: [PATCH 074/211] feat: newton crypto icon --- auth/assets/custom-icons/icons/newton.svg | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 auth/assets/custom-icons/icons/newton.svg diff --git a/auth/assets/custom-icons/icons/newton.svg b/auth/assets/custom-icons/icons/newton.svg new file mode 100644 index 0000000000..1f3939a582 --- /dev/null +++ b/auth/assets/custom-icons/icons/newton.svg @@ -0,0 +1,4 @@ + + + + From 00d0cfe72c06c8a12e43422fabff59edaf2dfa35 Mon Sep 17 00:00:00 2001 From: httpjamesm <51917118+httpjamesm@users.noreply.github.com> Date: Thu, 8 Aug 2024 00:12:51 -0400 Subject: [PATCH 075/211] feat: add shakepay and newton icon metadata --- auth/assets/custom-icons/_data/custom-icons.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index 80f81b2887..41226ecdf4 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -608,6 +608,15 @@ ], "slug": "ynab", "hex": "3B5EDA" + }, + { + "title": "Shakepay", + "slug": "shakepay" + }, + { + "title": "Newton", + "altNames": ["Newton Crypto"], + "slug": "newton" } ] } From 08303d2bb61112b3c6458dd526567c747afa8b5a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 10:36:48 +0530 Subject: [PATCH 076/211] Outline --- .../new/photos/services/ml/cluster-new.ts | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 web/packages/new/photos/services/ml/cluster-new.ts diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts new file mode 100644 index 0000000000..e1adb96500 --- /dev/null +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -0,0 +1,50 @@ +import log from "@/base/log"; +import type { FaceIndex } from "./face"; + +/** + * A cluster is an set of faces. + * + * Each cluster has an id so that a Person (a set of clusters) can refer to it. + */ +export interface Cluster { + /** A unique nanoid to identify this cluster. */ + id: string; + /** + * An unordered set of ids of the faces that belong to the cluster. + * + * For ergonomics of transportation and persistence this is an array but it + * should conceptually be thought of as a set. + */ + faceIDs: string[]; +} + +/** + * A Person is a set of clusters, with some attached metadata. + * + * The person is the user visible concept. It consists of a set of clusters, + * each of which itself is a set of faces. + */ +export interface Person { + /** A unique nanoid to identify this person. */ + id: string; + /** + * An unordered set of ids of the clusters that belong to this person. + * For ergonomics of transportation and persistence this is an array but it + * should conceptually be thought of as a set. + */ + clusterIDs: string[]; +} + +/** + * Cluster faces into groups. + * + * [Note: Face clustering algorithm] + * + * 1. clusters = [] + * 2. For each face, find its nearest neighbour in the embedding space. If no + * such neighbour is found within our threshold, create a new c + */ +export const clusterFaces = (faceIndices: FaceIndex[]) => { + log.debug(() => ["Clustering", faceIndices]); + return undefined; +}; From 5cc84793542da0f2bcea304f1d51afb847f0cbb4 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 10:50:49 +0530 Subject: [PATCH 077/211] Outline 2 --- .../new/photos/services/ml/cluster-new.ts | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index e1adb96500..0f1abd153e 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -23,12 +23,23 @@ export interface Cluster { * * The person is the user visible concept. It consists of a set of clusters, * each of which itself is a set of faces. + * + * For ease of transportation, the Person entity on remote looks like + * + * { name, clusters: { cluster_id, face_ids }} + * + * That is, it has the clusters embedded within itself. */ export interface Person { /** A unique nanoid to identify this person. */ id: string; + /** + * An optional name assigned by the user to this person. + */ + name: string | undefined; /** * An unordered set of ids of the clusters that belong to this person. + * * For ergonomics of transportation and persistence this is an array but it * should conceptually be thought of as a set. */ @@ -40,9 +51,19 @@ export interface Person { * * [Note: Face clustering algorithm] * - * 1. clusters = [] - * 2. For each face, find its nearest neighbour in the embedding space. If no - * such neighbour is found within our threshold, create a new c + * 1. clusters = [] + * 2. For each face, find its nearest neighbour in the embedding space. If no + * such neighbour is found within our threshold, create a new cluster. + * 3. Otherwise assign this face to the same cluster as its nearest neighbour. + * + * [Note: Face clustering feedback] + * + * This user can tweak the output of the algorithm by providing feedback. They + * can perform the following actions: + * + * 1. Move a cluster from one person to another. + * 2. Break a cluster. + * */ export const clusterFaces = (faceIndices: FaceIndex[]) => { log.debug(() => ["Clustering", faceIndices]); From ce421eded48ef81c49dbdfa0466f16bafbc59aca Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 11:17:18 +0530 Subject: [PATCH 078/211] nanoids --- web/docs/dependencies.md | 3 +++ web/packages/base/id-worker.ts | 17 +++++++++++++ web/packages/base/id.ts | 24 +++++++++++++++++++ web/packages/base/package.json | 1 + .../new/photos/services/ml/cluster-new.ts | 9 +++++-- web/yarn.lock | 7 +++++- 6 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 web/packages/base/id-worker.ts create mode 100644 web/packages/base/id.ts diff --git a/web/docs/dependencies.md b/web/docs/dependencies.md index c28824b288..efdbf617ce 100644 --- a/web/docs/dependencies.md +++ b/web/docs/dependencies.md @@ -193,6 +193,9 @@ For more details, see [translations.md](translations.md). - [zod](https://github.com/colinhacks/zod) is used for runtime typechecking (e.g. verifying that API responses match the expected TypeScript shape). +- [nanoid](https://github.com/ai/nanoid) is used for generating unique + identifiers. + - [debounce](https://github.com/sindresorhus/debounce) and its promise-supporting sibling [pDebounce](https://github.com/sindresorhus/p-debounce) are used for diff --git a/web/packages/base/id-worker.ts b/web/packages/base/id-worker.ts new file mode 100644 index 0000000000..593e2d236e --- /dev/null +++ b/web/packages/base/id-worker.ts @@ -0,0 +1,17 @@ +import { customAlphabet } from "nanoid/non-secure"; +import { alphabet } from "./id"; + +const nanoid = customAlphabet(alphabet, 22); + +/** + * This is a variant of the regular {@link newID} that can be used in web + * workers. + * + * Web workers don't have access to a secure random generator, so we need to use + * the non-secure variant. + * https://github.com/ai/nanoid?tab=readme-ov-file#web-workers + * + * For many of our use cases, where we're not using these IDs for cryptographic + * operations, this is okay. We also have an increased alphabet length. + */ +export const newNonSecureID = (prefix: string) => prefix + nanoid(); diff --git a/web/packages/base/id.ts b/web/packages/base/id.ts new file mode 100644 index 0000000000..5b9cebbbaa --- /dev/null +++ b/web/packages/base/id.ts @@ -0,0 +1,24 @@ +import { customAlphabet } from "nanoid"; + +/** + * Remove _ and - from the default set to have better looking IDs that can also + * be selected in the editor quickly ("-" prevents this), and which we can + * prefix unambigously ("_" is used for that). + * + * To compensate, increase length from the default of 21 to 22. + * + * To play around with these, use https://zelark.github.io/nano-id-cc/ + */ +export const alphabet = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +const nanoid = customAlphabet(alphabet, 22); + +/** + * Generate a new random identifier with the given prefix. + * + * Internally this uses [nanoids](https://github.com/ai/nanoid). + * + * See {@link newNonSecureID} for a variant that can be used in web workers. + */ +export const newID = (prefix: string) => prefix + nanoid(); diff --git a/web/packages/base/package.json b/web/packages/base/package.json index 86f21d8fdf..36bcf203f9 100644 --- a/web/packages/base/package.json +++ b/web/packages/base/package.json @@ -12,6 +12,7 @@ "i18next": "^23.11", "i18next-resources-to-backend": "^1.2", "is-electron": "^2.2", + "nanoid": "^5.0.7", "next": "^14.2", "react": "^18", "react-dom": "^18", diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index 0f1abd153e..bad2652f7d 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -7,7 +7,9 @@ import type { FaceIndex } from "./face"; * Each cluster has an id so that a Person (a set of clusters) can refer to it. */ export interface Cluster { - /** A unique nanoid to identify this cluster. */ + /** + * A randomly generated ID to uniquely identify this cluster. + */ id: string; /** * An unordered set of ids of the faces that belong to the cluster. @@ -31,7 +33,9 @@ export interface Cluster { * That is, it has the clusters embedded within itself. */ export interface Person { - /** A unique nanoid to identify this person. */ + /** + * A randomly generated ID to uniquely identify this person. + */ id: string; /** * An optional name assigned by the user to this person. @@ -67,5 +71,6 @@ export interface Person { */ export const clusterFaces = (faceIndices: FaceIndex[]) => { log.debug(() => ["Clustering", faceIndices]); + const clusters: Cluster = [] return undefined; }; diff --git a/web/yarn.lock b/web/yarn.lock index caf41595f2..04405053f3 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -3374,7 +3374,7 @@ libsodium-wrappers@0.7.9: dependencies: libsodium "^0.7.0" -libsodium@0.7.9, libsodium@^0.7.0: +libsodium@^0.7.0: version "0.7.9" resolved "https://registry.yarnpkg.com/libsodium/-/libsodium-0.7.9.tgz#4bb7bcbf662ddd920d8795c227ae25bbbfa3821b" integrity sha512-gfeADtR4D/CM0oRUviKBViMGXZDgnFdMKMzHsvBdqLBHd9ySi6EtYnmuhHVDDYgYpAO8eU8hEY+F8vIUAPh08A== @@ -3590,6 +3590,11 @@ nanoid@^3.3.6, nanoid@^3.3.7: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== +nanoid@^5.0.7: + version "5.0.7" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-5.0.7.tgz#6452e8c5a816861fd9d2b898399f7e5fd6944cc6" + integrity sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ== + natural-compare@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" From 395fe16d8a60492c5222c0326bce080096a9f50f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 11:51:01 +0530 Subject: [PATCH 079/211] dp --- .../new/photos/services/ml/cluster-new.ts | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index bad2652f7d..a0b66079f9 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -1,5 +1,6 @@ import log from "@/base/log"; import type { FaceIndex } from "./face"; +import { dotProduct } from "./math"; /** * A cluster is an set of faces. @@ -71,6 +72,37 @@ export interface Person { */ export const clusterFaces = (faceIndices: FaceIndex[]) => { log.debug(() => ["Clustering", faceIndices]); - const clusters: Cluster = [] + + const faces = [...faceIDAndEmbeddings(faceIndices)]; + + const clusters: Cluster = []; + for (const [i, fi] of faces.entries()) { + for (let j = i + 1; j < faces.length; j++) { + // Can't find a better way for avoiding the null assertion. + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const fj = faces[j]!; + + // TODO: The distance metric, the thresholds are placeholders. + + // The vectors are already normalized, so we can directly use their + // dot product as their cosine similarity. + const csim = dotProduct(fi.embedding, fj.embedding); + if (csim > 0.5) { + + } + } + } return undefined; }; + +/** + * A generator function that returns a stream of {faceID, embedding} values, + * flattening all the all the faces present in the given {@link faceIndices}. + */ +function* faceIDAndEmbeddings(faceIndices: FaceIndex[]) { + for (const fi of faceIndices) { + for (const f of fi.faces) { + yield { faceID: f.faceID, embedding: f.embedding }; + } + } +} From 2eb0cb348775ead1c0a8c1511442c9172d9b4475 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:04:05 +0530 Subject: [PATCH 080/211] Update DB Schema --- server/ente/filedata/filedata.go | 2 +- server/migrations/89_file_data_table.down.sql | 1 + server/migrations/89_file_data_table.up.sql | 49 +++++++++++-------- server/pkg/repo/filedata/repository.go | 29 ++++++----- 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go index 20ff30aea1..2ddc373f16 100644 --- a/server/ente/filedata/filedata.go +++ b/server/ente/filedata/filedata.go @@ -43,7 +43,7 @@ type Row struct { InflightReplicas []string PendingSync bool IsDeleted bool - LastSyncTime int64 + SyncLockedTill int64 CreatedAt int64 UpdatedAt int64 } diff --git a/server/migrations/89_file_data_table.down.sql b/server/migrations/89_file_data_table.down.sql index 024a1da670..bdf2a8717b 100644 --- a/server/migrations/89_file_data_table.down.sql +++ b/server/migrations/89_file_data_table.down.sql @@ -1,5 +1,6 @@ DROP INDEX IF EXISTS idx_file_data_user_type_deleted; +DROP INDEX IF EXISTS idx_file_data_last_sync_time; DROP TABLE IF EXISTS file_data; diff --git a/server/migrations/89_file_data_table.up.sql b/server/migrations/89_file_data_table.up.sql index 36d66587a6..c0feab6e64 100644 --- a/server/migrations/89_file_data_table.up.sql +++ b/server/migrations/89_file_data_table.up.sql @@ -1,40 +1,45 @@ -ALTER TABLE temp_objects ADD COLUMN IF NOT EXISTS bucket_id s3region; +ALTER TABLE temp_objects +ADD COLUMN IF NOT EXISTS bucket_id s3region; ALTER TYPE OBJECT_TYPE ADD VALUE 'derivedMeta'; ALTER TYPE s3region ADD VALUE 'b5'; --- Create the derived table +-- Create the file_data table CREATE TABLE IF NOT EXISTS file_data ( - file_id BIGINT NOT NULL, - user_id BIGINT NOT NULL, - data_type OBJECT_TYPE NOT NULL, - size BIGINT NOT NULL, - latest_bucket s3region NOT NULL, - replicated_buckets s3region[] NOT NULL DEFAULT '{}', + file_id BIGINT NOT NULL, + user_id BIGINT NOT NULL, + data_type OBJECT_TYPE NOT NULL, + size BIGINT NOT NULL, + latest_bucket s3region NOT NULL, + replicated_buckets s3region[] NOT NULL DEFAULT '{}', -- following field contains list of buckets from where we need to delete the data as the given data_type will not longer be persisted in that dc - delete_from_buckets s3region[] NOT NULL DEFAULT '{}', - inflight_rep_buckets s3region[] NOT NULL DEFAULT '{}', - pending_sync BOOLEAN NOT NULL DEFAULT false, - is_deleted BOOLEAN NOT NULL DEFAULT false, - last_sync_time BIGINT NOT NULL DEFAULT 0, - created_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), - updated_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), + delete_from_buckets s3region[] NOT NULL DEFAULT '{}', + inflight_rep_buckets s3region[] NOT NULL DEFAULT '{}', + is_deleted BOOLEAN NOT NULL DEFAULT false, + pending_sync BOOLEAN NOT NULL DEFAULT false, + sync_locked_till BIGINT NOT NULL DEFAULT 0, + created_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), + updated_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), PRIMARY KEY (file_id, data_type) ); -- Add index for user_id and data_type for efficient querying CREATE INDEX idx_file_data_user_type_deleted ON file_data (user_id, data_type, is_deleted) INCLUDE (file_id, size); +CREATE INDEX idx_file_data_pending_sync_locked_till ON file_data (is_deleted, sync_locked_till) where pending_sync = true; CREATE OR REPLACE FUNCTION ensure_no_common_entries() - RETURNS TRIGGER AS $$ + RETURNS TRIGGER AS +$$ DECLARE - all_buckets s3region[]; + all_buckets s3region[]; duplicate_buckets s3region[]; BEGIN -- Combine all bucket IDs into a single array - all_buckets := ARRAY[NEW.latest_bucket] || NEW.replicated_buckets || NEW.delete_from_buckets || NEW.inflight_rep_buckets; + all_buckets := ARRAY [NEW.latest_bucket] || NEW.replicated_buckets || NEW.delete_from_buckets || + NEW.inflight_rep_buckets; -- Find duplicate bucket IDs - SELECT ARRAY_AGG(DISTINCT bucket) INTO duplicate_buckets + SELECT ARRAY_AGG(DISTINCT bucket) + INTO duplicate_buckets FROM unnest(all_buckets) bucket GROUP BY bucket HAVING COUNT(*) > 1; @@ -50,6 +55,8 @@ END; $$ LANGUAGE plpgsql; CREATE TRIGGER check_no_common_entries - BEFORE INSERT OR UPDATE ON file_data - FOR EACH ROW EXECUTE FUNCTION ensure_no_common_entries(); + BEFORE INSERT OR UPDATE + ON file_data + FOR EACH ROW +EXECUTE FUNCTION ensure_no_common_entries(); diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go index 75c409a6c6..21208697bd 100644 --- a/server/pkg/repo/filedata/repository.go +++ b/server/pkg/repo/filedata/repository.go @@ -52,7 +52,7 @@ func (r *Repository) InsertOrUpdate(ctx context.Context, data filedata.Row) erro } func (r *Repository) GetFilesData(ctx context.Context, oType ente.ObjectType, fileIDs []int64) ([]filedata.Row, error) { - rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, inflight_rep_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at + rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, inflight_rep_buckets, pending_sync, is_deleted, sync_locked_till, created_at, updated_at FROM file_data WHERE data_type = $1 AND file_id = ANY($2)`, string(oType), pq.Array(fileIDs)) if err != nil { @@ -62,7 +62,7 @@ func (r *Repository) GetFilesData(ctx context.Context, oType ente.ObjectType, fi } func (r *Repository) GetFileData(ctx context.Context, fileIDs int64) ([]filedata.Row, error) { - rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets,inflight_rep_buckets, pending_sync, is_deleted, last_sync_time, created_at, updated_at + rows, err := r.DB.QueryContext(ctx, `SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets,inflight_rep_buckets, pending_sync, is_deleted, sync_locked_till, created_at, updated_at FROM file_data WHERE file_id = $1`, fileIDs) if err != nil { @@ -96,17 +96,17 @@ func (r *Repository) AddBucket(row filedata.Row, bucketID string, columnName str func (r *Repository) RemoveBucket(row filedata.Row, bucketID string, columnName string) error { query := fmt.Sprintf(` - UPDATE file_data - SET %s = array( - SELECT DISTINCT elem FROM unnest( - array_remove( - file_data.%s, - $1 - ) - ) AS elem - WHERE elem IS NOT NULL - ) - WHERE file_id = $2 AND data_type = $3 and user_id = $4`, columnName, columnName) + UPDATE file_data + SET %s = array( + SELECT DISTINCT elem FROM unnest( + array_remove( + file_data.%s, + $1 + ) + ) AS elem + WHERE elem IS NOT NULL + ) + WHERE file_id = $2 AND data_type = $3 and user_id = $4`, columnName, columnName) result, err := r.DB.Exec(query, bucketID, row.FileID, string(row.Type), row.UserID) if err != nil { return stacktrace.Propagate(err, "failed to remove bucket from "+columnName) @@ -173,14 +173,13 @@ WHERE file_id = $1 AND data_type = $2 AND latest_bucket = $3 AND user_id = $4 AN return stacktrace.NewError("file data not deleted") } return nil - } func convertRowsToFilesData(rows *sql.Rows) ([]filedata.Row, error) { var filesData []filedata.Row for rows.Next() { var fileData filedata.Row - err := rows.Scan(&fileData.FileID, &fileData.UserID, &fileData.Type, &fileData.Size, &fileData.LatestBucket, pq.Array(&fileData.ReplicatedBuckets), pq.Array(&fileData.DeleteFromBuckets), pq.Array(&fileData.InflightReplicas), &fileData.PendingSync, &fileData.IsDeleted, &fileData.LastSyncTime, &fileData.CreatedAt, &fileData.UpdatedAt) + err := rows.Scan(&fileData.FileID, &fileData.UserID, &fileData.Type, &fileData.Size, &fileData.LatestBucket, pq.Array(&fileData.ReplicatedBuckets), pq.Array(&fileData.DeleteFromBuckets), pq.Array(&fileData.InflightReplicas), &fileData.PendingSync, &fileData.IsDeleted, &fileData.SyncLockedTill, &fileData.CreatedAt, &fileData.UpdatedAt) if err != nil { return nil, stacktrace.Propagate(err, "") } From e31f0b042dba08ca8ab4f38de20f28df977d0b85 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 12:01:01 +0530 Subject: [PATCH 081/211] Sketch --- .../new/photos/services/ml/cluster-new.ts | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index a0b66079f9..ffc1083804 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -1,3 +1,4 @@ +import { newNonSecureID } from "@/base/id-worker"; import log from "@/base/log"; import type { FaceIndex } from "./face"; import { dotProduct } from "./math"; @@ -71,27 +72,46 @@ export interface Person { * */ export const clusterFaces = (faceIndices: FaceIndex[]) => { - log.debug(() => ["Clustering", faceIndices]); + const t = Date.now(); const faces = [...faceIDAndEmbeddings(faceIndices)]; - const clusters: Cluster = []; + const clusters: Cluster[] = []; + const clusterIndexByFaceID = new Map(); for (const [i, fi] of faces.entries()) { - for (let j = i + 1; j < faces.length; j++) { + let j = i + 1; + while (j < faces.length) { // Can't find a better way for avoiding the null assertion. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const fj = faces[j]!; + const fj = faces[j++]!; - // TODO: The distance metric, the thresholds are placeholders. + // TODO-ML: The distance metric and the thresholds are placeholders. // The vectors are already normalized, so we can directly use their // dot product as their cosine similarity. const csim = dotProduct(fi.embedding, fj.embedding); if (csim > 0.5) { - + // Found a neighbour near enough. Add this face to the + // neighbour's cluster and call it a day. } } + if (j == faces.length) { + // We didn't find a neighbour. Create a new cluster with this face. + const cluster = { + id: newNonSecureID("cluster_"), + faceIDs: [fi.faceID], + }; + clusters.push(cluster); + clusterIndexByFaceID.set(fi.faceID, clusters.length); + } } + + log.debug(() => ["ml/cluster", { faces, clusters, clusterIndexByFaceID }]); + log.debug( + () => + `Clustered ${faces.length} faces into ${clusters.length} clusters (${Date.now() - t} ms)`, + ); + return undefined; }; From d53d39b40081de832d5ab69620238427b9843c0a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 12:27:19 +0530 Subject: [PATCH 082/211] Loop --- web/packages/new/photos/services/ml/cluster-new.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index ffc1083804..ee46a31abc 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -1,5 +1,6 @@ import { newNonSecureID } from "@/base/id-worker"; import log from "@/base/log"; +import { ensure } from "@/utils/ensure"; import type { FaceIndex } from "./face"; import { dotProduct } from "./math"; @@ -80,10 +81,10 @@ export const clusterFaces = (faceIndices: FaceIndex[]) => { const clusterIndexByFaceID = new Map(); for (const [i, fi] of faces.entries()) { let j = i + 1; - while (j < faces.length) { + for (; j < faces.length; j++) { // Can't find a better way for avoiding the null assertion. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const fj = faces[j++]!; + const fj = faces[j]!; // TODO-ML: The distance metric and the thresholds are placeholders. @@ -93,6 +94,10 @@ export const clusterFaces = (faceIndices: FaceIndex[]) => { if (csim > 0.5) { // Found a neighbour near enough. Add this face to the // neighbour's cluster and call it a day. + const ci = ensure(clusterIndexByFaceID.get(fj.faceID)); + clusters[ci]?.faceIDs.push(fi.faceID); + clusterIndexByFaceID.set(fi.faceID, ci); + break; } } if (j == faces.length) { From c784831dedefc77607bd49e6641bc9ec4ee25b6a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 12:35:04 +0530 Subject: [PATCH 083/211] Test --- web/packages/new/photos/services/ml/cluster-new.ts | 4 ++-- web/packages/new/photos/services/ml/db.ts | 8 ++++++++ web/packages/new/photos/services/ml/index.ts | 14 +++++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index ee46a31abc..c628ffab71 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -72,10 +72,10 @@ export interface Person { * 2. Break a cluster. * */ -export const clusterFaces = (faceIndices: FaceIndex[]) => { +export const clusterFaces = (faceIndexes: FaceIndex[]) => { const t = Date.now(); - const faces = [...faceIDAndEmbeddings(faceIndices)]; + const faces = [...faceIDAndEmbeddings(faceIndexes)]; const clusters: Cluster[] = []; const clusterIndexByFaceID = new Map(); diff --git a/web/packages/new/photos/services/ml/db.ts b/web/packages/new/photos/services/ml/db.ts index a648aa8f06..3fe18d0731 100644 --- a/web/packages/new/photos/services/ml/db.ts +++ b/web/packages/new/photos/services/ml/db.ts @@ -231,6 +231,14 @@ export const faceIndex = async (fileID: number) => { return db.get("face-index", fileID); }; +/** + * Return all face indexes present locally. + */ +export const faceIndexes = async () => { + const db = await mlDB(); + return await db.getAll("face-index"); +}; + /** * Return all CLIP indexes present locally. */ diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 5b57dade21..784bf3fa52 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -5,6 +5,7 @@ import { isDesktop } from "@/base/app"; import { blobCache } from "@/base/blob-cache"; import { ensureElectron } from "@/base/electron"; +import { isDevBuild } from "@/base/env"; import log from "@/base/log"; import type { Electron } from "@/base/types/ipc"; import { ComlinkWorker } from "@/base/worker/comlink-worker"; @@ -17,9 +18,10 @@ import { isInternalUser } from "../feature-flags"; import { getRemoteFlag, updateRemoteFlag } from "../remote-store"; import type { UploadItem } from "../upload/types"; import { regenerateFaceCrops } from "./crop"; -import { clearMLDB, faceIndex, indexableAndIndexedCounts } from "./db"; +import { clearMLDB, faceIndex, faceIndexes, indexableAndIndexedCounts } from "./db"; import { MLWorker } from "./worker"; import type { CLIPMatches } from "./worker-types"; +import { clusterFaces } from "./cluster-new"; /** * In-memory flag that tracks if ML is enabled. @@ -279,6 +281,16 @@ export const indexNewUpload = (enteFile: EnteFile, uploadItem: UploadItem) => { void worker().then((w) => w.onUpload(enteFile, uploadItem)); }; +/** + * WIP! Don't enable, dragon eggs are hatching here. + */ +export const wipCluster = async () => { + if (!isDevBuild || !(await isInternalUser())) return; + if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return; + + clusterFaces(await faceIndexes()); +}; + export type MLStatus = | { phase: "disabled" /* The ML remote flag is off */ } | { From 1c8512ad81c421f58df9e4c707b5bd8c00af7972 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 12:38:33 +0530 Subject: [PATCH 084/211] wip harness --- web/packages/new/photos/services/ml/index.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 784bf3fa52..55e482b464 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -17,11 +17,16 @@ import { proxy, transfer } from "comlink"; import { isInternalUser } from "../feature-flags"; import { getRemoteFlag, updateRemoteFlag } from "../remote-store"; import type { UploadItem } from "../upload/types"; +import { clusterFaces } from "./cluster-new"; import { regenerateFaceCrops } from "./crop"; -import { clearMLDB, faceIndex, faceIndexes, indexableAndIndexedCounts } from "./db"; +import { + clearMLDB, + faceIndex, + faceIndexes, + indexableAndIndexedCounts, +} from "./db"; import { MLWorker } from "./worker"; import type { CLIPMatches } from "./worker-types"; -import { clusterFaces } from "./cluster-new"; /** * In-memory flag that tracks if ML is enabled. @@ -257,6 +262,8 @@ const mlSync = async () => { triggerStatusUpdate(); if (_isMLEnabled) void worker().then((w) => w.sync()); + // TODO-ML + if (_isMLEnabled) void wipCluster(); }; /** From 5ad1bacf3efc65b87c2e06520c4e886b9e2dc0e8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 12:46:13 +0530 Subject: [PATCH 085/211] Swap --- .../new/photos/services/ml/cluster-new.ts | 29 ++++++++++--------- web/yarn.lock | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index c628ffab71..c5e3514a67 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -59,9 +59,10 @@ export interface Person { * [Note: Face clustering algorithm] * * 1. clusters = [] - * 2. For each face, find its nearest neighbour in the embedding space. If no - * such neighbour is found within our threshold, create a new cluster. - * 3. Otherwise assign this face to the same cluster as its nearest neighbour. + * 2. For each face, find its nearest neighbour in the embedding space from + * amongst the faces that have already been clustered. + * 3. If no such neighbour is found within our threshold, create a new cluster. + * 4. Otherwise assign this face to the same cluster as its nearest neighbour. * * [Note: Face clustering feedback] * @@ -79,35 +80,35 @@ export const clusterFaces = (faceIndexes: FaceIndex[]) => { const clusters: Cluster[] = []; const clusterIndexByFaceID = new Map(); - for (const [i, fi] of faces.entries()) { - let j = i + 1; - for (; j < faces.length; j++) { + for (const [i, { faceID, embedding }] of faces.entries()) { + let j = 0; + for (; j < i; j++) { // Can't find a better way for avoiding the null assertion. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - const fj = faces[j]!; + const n = faces[j]!; // TODO-ML: The distance metric and the thresholds are placeholders. // The vectors are already normalized, so we can directly use their // dot product as their cosine similarity. - const csim = dotProduct(fi.embedding, fj.embedding); + const csim = dotProduct(embedding, n.embedding); if (csim > 0.5) { // Found a neighbour near enough. Add this face to the // neighbour's cluster and call it a day. - const ci = ensure(clusterIndexByFaceID.get(fj.faceID)); - clusters[ci]?.faceIDs.push(fi.faceID); - clusterIndexByFaceID.set(fi.faceID, ci); + const ci = ensure(clusterIndexByFaceID.get(n.faceID)); + clusters[ci]?.faceIDs.push(faceID); + clusterIndexByFaceID.set(faceID, ci); break; } } - if (j == faces.length) { + if (j == i) { // We didn't find a neighbour. Create a new cluster with this face. const cluster = { id: newNonSecureID("cluster_"), - faceIDs: [fi.faceID], + faceIDs: [faceID], }; clusters.push(cluster); - clusterIndexByFaceID.set(fi.faceID, clusters.length); + clusterIndexByFaceID.set(faceID, clusters.length); } } diff --git a/web/yarn.lock b/web/yarn.lock index 04405053f3..90cbd1d0b5 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -3374,7 +3374,7 @@ libsodium-wrappers@0.7.9: dependencies: libsodium "^0.7.0" -libsodium@^0.7.0: +libsodium@0.7.9, libsodium@^0.7.0: version "0.7.9" resolved "https://registry.yarnpkg.com/libsodium/-/libsodium-0.7.9.tgz#4bb7bcbf662ddd920d8795c227ae25bbbfa3821b" integrity sha512-gfeADtR4D/CM0oRUviKBViMGXZDgnFdMKMzHsvBdqLBHd9ySi6EtYnmuhHVDDYgYpAO8eU8hEY+F8vIUAPh08A== From 50f6fd7440630708cb3c8c1bee2bb1d8d612bb54 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:51:42 +0530 Subject: [PATCH 086/211] Add request to get preview url --- server/ente/filedata/filedata.go | 12 ++++++++++++ server/ente/filedata/path.go | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go index 2ddc373f16..f9cfccd28e 100644 --- a/server/ente/filedata/filedata.go +++ b/server/ente/filedata/filedata.go @@ -72,3 +72,15 @@ type S3FileMetadata struct { DecryptionHeader string `json:"header"` Client string `json:"client"` } + +type GetPreviewUrlRequest struct { + FileID int64 `form:"fileID" binding:"required"` + Type ente.ObjectType `form:"type" binding:"required"` +} + +func (g *GetPreviewUrlRequest) Validate() error { + if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage { + return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) + } + return nil +} diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.go index 5dd4f616ef..fce63bad09 100644 --- a/server/ente/filedata/path.go +++ b/server/ente/filedata/path.go @@ -25,6 +25,17 @@ func AllObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string { } } +func PreviewUrl(fileID int64, ownerID int64, oType ente.ObjectType) string { + switch oType { + case ente.PreviewVideo: + return previewVideoPath(fileID, ownerID) + case ente.PreviewImage: + return previewImagePath(fileID, ownerID) + default: + panic(fmt.Sprintf("object type %s is not supported", oType)) + } +} + func previewVideoPath(fileID int64, ownerID int64) string { return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.PreviewVideo)) } From 272d17615ec44087998cfc20c7f18e6c382c3ff0 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 8 Aug 2024 13:03:34 +0530 Subject: [PATCH 087/211] [server] Remove fileData cleanup via queue --- server/cmd/museum/main.go | 13 +- server/pkg/controller/filedata/delete.go | 131 ++++++++------------ server/pkg/controller/filedata/replicate.go | 6 + server/pkg/repo/object.go | 7 +- server/pkg/repo/queue.go | 2 - 5 files changed, 70 insertions(+), 89 deletions(-) create mode 100644 server/pkg/controller/filedata/replicate.go diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index 10b5a7fc6c..8c9c88f304 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -706,11 +706,11 @@ func main() { publicAPI.GET("/offers/black-friday", offerHandler.GetBlackFridayOffers) setKnownAPIs(server.Routes()) - setupAndStartBackgroundJobs(objectCleanupController, replicationController3) + setupAndStartBackgroundJobs(objectCleanupController, replicationController3, fileDataCtrl) setupAndStartCrons( userAuthRepo, publicCollectionRepo, twoFactorRepo, passkeysRepo, fileController, taskLockingRepo, emailNotificationCtrl, trashController, pushController, objectController, dataCleanupController, storageBonusCtrl, - embeddingController, fileDataCtrl, healthCheckHandler, kexCtrl, castDb) + embeddingController, healthCheckHandler, kexCtrl, castDb) // Create a new collector, the name will be used as a label on the metrics collector := sqlstats.NewStatsCollector("prod_db", db) @@ -816,6 +816,7 @@ func setupDatabase() *sql.DB { func setupAndStartBackgroundJobs( objectCleanupController *controller.ObjectCleanupController, replicationController3 *controller.ReplicationController3, + fileDataCtrl *filedata.Controller, ) { isReplicationEnabled := viper.GetBool("replication.enabled") if isReplicationEnabled { @@ -823,9 +824,14 @@ func setupAndStartBackgroundJobs( if err != nil { log.Warnf("Could not start replication v3: %s", err) } + err = fileDataCtrl.StartReplication() + if err != nil { + log.Warnf("Could not start fileData replication: %s", err) + } } else { log.Info("Skipping Replication as replication is disabled") } + fileDataCtrl.StartDataDeletion() // Start data deletion for file data; objectCleanupController.StartRemovingUnreportedObjects() objectCleanupController.StartClearingOrphanObjects() @@ -839,7 +845,6 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, publicCollectionR dataCleanupCtrl *dataCleanupCtrl.DeleteUserCleanupController, storageBonusCtrl *storagebonus.Controller, embeddingCtrl *embeddingCtrl.Controller, - fileDataCtrl *filedata.Controller, healthCheckHandler *api.HealthCheckHandler, kexCtrl *kexCtrl.Controller, castDb castRepo.Repository) { @@ -883,10 +888,8 @@ func setupAndStartCrons(userAuthRepo *repo.UserAuthRepository, publicCollectionR schedule(c, "@every 2m", func() { fileController.CleanupDeletedFiles() }) - fileDataCtrl.CleanUpDeletedFileData() schedule(c, "@every 101s", func() { embeddingCtrl.CleanupDeletedEmbeddings() - fileDataCtrl.CleanUpDeletedFileData() }) schedule(c, "@every 10m", func() { diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go index 1673e7c0fc..799844d06b 100644 --- a/server/pkg/controller/filedata/delete.go +++ b/server/pkg/controller/filedata/delete.go @@ -4,93 +4,58 @@ import ( "context" "fmt" "github.com/ente-io/museum/ente/filedata" - "github.com/ente-io/museum/pkg/repo" fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata" - "github.com/ente-io/museum/pkg/utils/time" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" - "strconv" ) -// CleanUpDeletedFileData clears associated file data from the object store -func (c *Controller) CleanUpDeletedFileData() { +// StartDataDeletion clears associated file data from the object store +func (c *Controller) StartDataDeletion() { log.Info("Cleaning up deleted file data") - if c.cleanupCronRunning { - log.Info("Skipping CleanUpDeletedFileData cron run as another instance is still running") - return - } - c.cleanupCronRunning = true - defer func() { - c.cleanupCronRunning = false - }() - items, err := c.QueueRepo.GetItemsReadyForDeletion(repo.DeleteFileDataQueue, 200) - if err != nil { - log.WithError(err).Error("Failed to fetch items from queue") - return - } - for _, i := range items { - c.deleteFileData(i) - } + // todo: start goroutine workers to delete data + } -func (c *Controller) deleteFileData(qItem repo.QueueItem) { - lockName := fmt.Sprintf("FileDataDelete:%s", qItem.Item) - lockStatus, err := c.TaskLockingRepo.AcquireLock(lockName, time.MicrosecondsAfterHours(1), c.HostName) - ctxLogger := log.WithField("item", qItem.Item).WithField("queue_id", qItem.Id) - if err != nil || !lockStatus { - ctxLogger.Warn("unable to acquire lock") - return - } - defer func() { - err = c.TaskLockingRepo.ReleaseLock(lockName) - if err != nil { - ctxLogger.Errorf("Error while releasing lock %s", err) - } - }() - ctxLogger.Debug("Deleting all file data") - fileID, _ := strconv.ParseInt(qItem.Item, 10, 64) +func (c *Controller) DeleteFileData(fileID int64) error { ownerID, err := c.FileRepo.GetOwnerID(fileID) if err != nil { - ctxLogger.WithError(err).Error("Failed to fetch ownerID") - return + return err } rows, err := c.Repo.GetFileData(context.Background(), fileID) if err != nil { - ctxLogger.WithError(err).Error("Failed to fetch datacenters") - return + return err } for i := range rows { fileDataRow := rows[i] + ctxLogger := log.WithField("file_id", fileDataRow.DeleteFromBuckets).WithField("type", fileDataRow.Type).WithField("user_id", fileDataRow.UserID) objectKeys := filedata.AllObjects(fileID, ownerID, fileDataRow.Type) - // Delete from delete/stale buckets - for j := range fileDataRow.DeleteFromBuckets { - bucketID := fileDataRow.DeleteFromBuckets[j] - for k := range objectKeys { - err = c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKeys[k], bucketID) - if err != nil { - ctxLogger.WithError(err).Error("Failed to delete object from datacenter") - return - } - } - dbErr := c.Repo.RemoveBucket(fileDataRow, bucketID, fileDataRepo.DeletionColumn) - if dbErr != nil { - ctxLogger.WithError(dbErr).Error("Failed to remove from db") - return - } + bucketColumnMap := make(map[string]string) + bucketColumnMap, err = getMapOfbucketItToColumn(fileDataRow) + if err != nil { + ctxLogger.WithError(err).Error("Failed to get bucketColumnMap") + return err } - // Delete from replicated buckets - for j := range fileDataRow.ReplicatedBuckets { - bucketID := fileDataRow.ReplicatedBuckets[j] - for k := range objectKeys { - err = c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKeys[k], bucketID) + // Delete objects and remove buckets + for bucketID, columnName := range bucketColumnMap { + for _, objectKey := range objectKeys { + err := c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKey, bucketID) if err != nil { - ctxLogger.WithError(err).Error("Failed to delete object from datacenter") - return + ctxLogger.WithError(err).WithFields(logrus.Fields{ + "bucketID": bucketID, + "column": columnName, + "objectKey": objectKey, + }).Error("Failed to delete object from datacenter") + return err } } - dbErr := c.Repo.RemoveBucket(fileDataRow, bucketID, fileDataRepo.ReplicationColumn) + dbErr := c.Repo.RemoveBucket(fileDataRow, bucketID, columnName) if dbErr != nil { - ctxLogger.WithError(dbErr).Error("Failed to remove from db") - return + ctxLogger.WithError(dbErr).WithFields(logrus.Fields{ + "bucketID": bucketID, + "column": columnName, + }).Error("Failed to remove bucket from db") + return dbErr + } } // Delete from Latest bucket @@ -98,23 +63,37 @@ func (c *Controller) deleteFileData(qItem repo.QueueItem) { err = c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKeys[k], fileDataRow.LatestBucket) if err != nil { ctxLogger.WithError(err).Error("Failed to delete object from datacenter") - return + return err } } dbErr := c.Repo.DeleteFileData(context.Background(), fileDataRow) if dbErr != nil { ctxLogger.WithError(dbErr).Error("Failed to remove from db") - return + return err } } - if err != nil { - ctxLogger.WithError(err).Error("Failed delete data") - return + return nil +} + +func getMapOfbucketItToColumn(row filedata.Row) (map[string]string, error) { + bucketColumnMap := make(map[string]string) + for _, bucketID := range row.DeleteFromBuckets { + if existingColumn, exists := bucketColumnMap[bucketID]; exists { + return nil, fmt.Errorf("Duplicate DeleteFromBuckets ID found: %s in column %s", bucketID, existingColumn) + } + bucketColumnMap[bucketID] = fileDataRepo.DeletionColumn } - err = c.QueueRepo.DeleteItem(repo.DeleteFileDataQueue, qItem.Item) - if err != nil { - ctxLogger.WithError(err).Error("Failed to remove item from the queue") - return + for _, bucketID := range row.ReplicatedBuckets { + if existingColumn, exists := bucketColumnMap[bucketID]; exists { + return nil, fmt.Errorf("Duplicate ReplicatedBuckets ID found: %s in column %s", bucketID, existingColumn) + } + bucketColumnMap[bucketID] = fileDataRepo.ReplicationColumn + } + for _, bucketID := range row.InflightReplicas { + if existingColumn, exists := bucketColumnMap[bucketID]; exists { + return nil, fmt.Errorf("Duplicate InFlightBucketID found: %s in column %s", bucketID, existingColumn) + } + bucketColumnMap[bucketID] = fileDataRepo.InflightRepColumn } - ctxLogger.Info("Successfully deleted all file data") + return bucketColumnMap, nil } diff --git a/server/pkg/controller/filedata/replicate.go b/server/pkg/controller/filedata/replicate.go new file mode 100644 index 0000000000..7baa814199 --- /dev/null +++ b/server/pkg/controller/filedata/replicate.go @@ -0,0 +1,6 @@ +package filedata + +func (c *Controller) StartReplication() error { + // todo: implement replication logic + return nil +} diff --git a/server/pkg/repo/object.go b/server/pkg/repo/object.go index fc02e2d25c..e5e7e49996 100644 --- a/server/pkg/repo/object.go +++ b/server/pkg/repo/object.go @@ -148,7 +148,7 @@ func (repo *ObjectRepository) MarkObjectsAsDeletedForFileIDs(ctx context.Context for _, fileID := range fileIDs { embeddingsToBeDeleted = append(embeddingsToBeDeleted, strconv.FormatInt(fileID, 10)) } - _, err = tx.ExecContext(ctx, `UPDATE file_data SET is_deleted = TRUE WHERE file_id = ANY($1)`, pq.Array(fileIDs)) + _, err = tx.ExecContext(ctx, `UPDATE file_data SET is_deleted = TRUE, pending_sync = TRUE WHERE file_id = ANY($1)`, pq.Array(fileIDs)) if err != nil { return nil, stacktrace.Propagate(err, "") } @@ -158,11 +158,6 @@ func (repo *ObjectRepository) MarkObjectsAsDeletedForFileIDs(ctx context.Context return nil, stacktrace.Propagate(err, "") } - err = repo.QueueRepo.AddItems(ctx, tx, DeleteFileDataQueue, embeddingsToBeDeleted) - if err != nil { - return nil, stacktrace.Propagate(err, "") - } - _, err = tx.ExecContext(ctx, `UPDATE object_keys SET is_deleted = TRUE WHERE file_id = ANY($1)`, pq.Array(fileIDs)) if err != nil { return nil, stacktrace.Propagate(err, "") diff --git a/server/pkg/repo/queue.go b/server/pkg/repo/queue.go index e4800aea9c..49544dbc8c 100644 --- a/server/pkg/repo/queue.go +++ b/server/pkg/repo/queue.go @@ -23,7 +23,6 @@ var itemDeletionDelayInMinMap = map[string]int64{ DropFileEncMedataQueue: -1 * 24 * 60, // -ve value to ensure attributes are immediately removed DeleteObjectQueue: 45 * 24 * 60, // 45 days in minutes DeleteEmbeddingsQueue: -1 * 24 * 60, // -ve value to ensure embeddings are immediately removed - DeleteFileDataQueue: -1 * 24 * 60, // -ve value to ensure file-data is immediately removed TrashCollectionQueueV3: -1 * 24 * 60, // -ve value to ensure collections are immediately marked as trashed TrashEmptyQueue: -1 * 24 * 60, // -ve value to ensure empty trash request are processed in next cron run RemoveComplianceHoldQueue: -1 * 24 * 60, // -ve value to ensure compliance hold is removed in next cron run @@ -33,7 +32,6 @@ const ( DropFileEncMedataQueue string = "dropFileEncMetata" DeleteObjectQueue string = "deleteObject" DeleteEmbeddingsQueue string = "deleteEmbedding" - DeleteFileDataQueue string = "deleteFileData" OutdatedObjectsQueue string = "outdatedObject" // Deprecated: Keeping it till we clean up items from the queue DB. TrashCollectionQueue string = "trashCollection" From d26aafc5f4949d9afa0e48d9e6577ec84befa36a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 13:57:17 +0530 Subject: [PATCH 088/211] Flowchart --- web/apps/photos/src/utils/billing/index.ts | 45 ++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/web/apps/photos/src/utils/billing/index.ts b/web/apps/photos/src/utils/billing/index.ts index 1eecbac553..654ac1cb32 100644 --- a/web/apps/photos/src/utils/billing/index.ts +++ b/web/apps/photos/src/utils/billing/index.ts @@ -31,6 +31,51 @@ enum RESPONSE_STATUS { fail = "fail", } +export type PlanSelectionOutcome = + | "buyPlan" + | "updateSubscriptionToPlan" + | "cancelOnMobile" + | "contactSupport"; + +/** + * Return the outcome that should happen when the user selects a paid plan on + * the plan selection screen. + * + * @param subscription Their current subscription details. + */ +export const planSelectionOutcome = ( + subscription: Subscription | undefined, +) => { + // This shouldn't happen, but we need this case to handle missing types. + if (!subscription) return "buyPlan"; + + // The user is a on a free plan and can buy the plan they selected. + if (subscription.productID == "free") return "buyPlan"; + + // Their existing subscription has expired. They can buy a new plan. + if (subscription.expiryTime < Date.now() * 1000) return "buyPlan"; + + // -- The user already has an active subscription to a paid plan. + + // Using stripe + if (subscription.paymentProvider == "stripe") { + // Update their existing subscription to the new plan. + return "updateSubscriptionToPlan"; + } + + // Using one of the mobile app stores + if ( + subscription.paymentProvider == "appstore" || + subscription.paymentProvider == "playstore" + ) { + // They need to cancel first on the mobile app stores. + return "cancelOnMobile"; + } + + // Some other bespoke case. They should contact support. + return "contactSupport"; +}; + export function hasPaidSubscription(subscription: Subscription) { return ( subscription && From 488c239cf2c9c510654b4e353212453f1013ddca Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 13:59:44 +0530 Subject: [PATCH 089/211] Use --- .../components/pages/gallery/PlanSelector.tsx | 105 +++++++++--------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/web/apps/photos/src/components/pages/gallery/PlanSelector.tsx b/web/apps/photos/src/components/pages/gallery/PlanSelector.tsx index 3879d3667a..c016bbe0dd 100644 --- a/web/apps/photos/src/components/pages/gallery/PlanSelector.tsx +++ b/web/apps/photos/src/components/pages/gallery/PlanSelector.tsx @@ -39,7 +39,6 @@ import { cancelSubscription, getLocalUserSubscription, hasAddOnBonus, - hasMobileSubscription, hasPaidSubscription, hasStripeSubscription, isOnFreePlan, @@ -49,6 +48,7 @@ import { isUserSubscribedPlan, manageFamilyMethod, planForSubscription, + planSelectionOutcome, updatePaymentMethod, updateSubscription, } from "utils/billing"; @@ -177,58 +177,63 @@ function PlanSelectorCard(props: PlanSelectorCardProps) { }, []); async function onPlanSelect(plan: Plan) { - if ( - !hasPaidSubscription(subscription) && - !isSubscriptionCancelled(subscription) - ) { - try { - props.setLoading(true); - await billingService.buySubscription(plan.stripeID); - } catch (e) { - props.setLoading(false); + switch (planSelectionOutcome(subscription)) { + case "buyPlan": + try { + props.setLoading(true); + await billingService.buySubscription(plan.stripeID); + } catch (e) { + props.setLoading(false); + appContext.setDialogMessage({ + title: t("ERROR"), + content: t("SUBSCRIPTION_PURCHASE_FAILED"), + close: { variant: "critical" }, + }); + } + break; + + case "updateSubscriptionToPlan": appContext.setDialogMessage({ - title: t("ERROR"), - content: t("SUBSCRIPTION_PURCHASE_FAILED"), - close: { variant: "critical" }, + title: t("update_subscription_title"), + content: t("UPDATE_SUBSCRIPTION_MESSAGE"), + proceed: { + text: t("UPDATE_SUBSCRIPTION"), + action: updateSubscription.bind( + null, + plan, + appContext.setDialogMessage, + props.setLoading, + props.closeModal, + ), + variant: "accent", + }, + close: { text: t("cancel") }, }); - } - } else if (hasStripeSubscription(subscription)) { - appContext.setDialogMessage({ - title: t("update_subscription_title"), - content: t("UPDATE_SUBSCRIPTION_MESSAGE"), - proceed: { - text: t("UPDATE_SUBSCRIPTION"), - action: updateSubscription.bind( - null, - plan, - appContext.setDialogMessage, - props.setLoading, - props.closeModal, + break; + + case "cancelOnMobile": + appContext.setDialogMessage({ + title: t("CANCEL_SUBSCRIPTION_ON_MOBILE"), + content: t("CANCEL_SUBSCRIPTION_ON_MOBILE_MESSAGE"), + close: { variant: "secondary" }, + }); + break; + + case "contactSupport": + appContext.setDialogMessage({ + title: t("MANAGE_PLAN"), + content: ( + , + }} + values={{ emailID: "support@ente.io" }} + /> ), - variant: "accent", - }, - close: { text: t("cancel") }, - }); - } else if (hasMobileSubscription(subscription)) { - appContext.setDialogMessage({ - title: t("CANCEL_SUBSCRIPTION_ON_MOBILE"), - content: t("CANCEL_SUBSCRIPTION_ON_MOBILE_MESSAGE"), - close: { variant: "secondary" }, - }); - } else { - appContext.setDialogMessage({ - title: t("MANAGE_PLAN"), - content: ( - , - }} - values={{ emailID: "support@ente.io" }} - /> - ), - close: { variant: "secondary" }, - }); + close: { variant: "secondary" }, + }); + break; } } From a3c51044c4565c91d729af94fbc8921e13bdde8b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 14:02:24 +0530 Subject: [PATCH 090/211] Unused --- web/apps/photos/src/utils/billing/index.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/web/apps/photos/src/utils/billing/index.ts b/web/apps/photos/src/utils/billing/index.ts index 654ac1cb32..50366e0407 100644 --- a/web/apps/photos/src/utils/billing/index.ts +++ b/web/apps/photos/src/utils/billing/index.ts @@ -13,8 +13,6 @@ import { getSubscriptionPurchaseSuccessMessage } from "utils/ui"; import { getTotalFamilyUsage, isPartOfFamily } from "utils/user/family"; const PAYMENT_PROVIDER_STRIPE = "stripe"; -const PAYMENT_PROVIDER_APPSTORE = "appstore"; -const PAYMENT_PROVIDER_PLAYSTORE = "playstore"; const FREE_PLAN = "free"; const THIRTY_DAYS_IN_MICROSECONDS = 30 * 24 * 60 * 60 * 1000 * 1000; @@ -137,15 +135,6 @@ export function hasStripeSubscription(subscription: Subscription) { ); } -export function hasMobileSubscription(subscription: Subscription) { - return ( - hasPaidSubscription(subscription) && - subscription.paymentProvider.length > 0 && - (subscription.paymentProvider === PAYMENT_PROVIDER_APPSTORE || - subscription.paymentProvider === PAYMENT_PROVIDER_PLAYSTORE) - ); -} - export function hasExceededStorageQuota(userDetails: UserDetails) { const bonusStorage = userDetails.storageBonus ?? 0; if (isPartOfFamily(userDetails.familyData)) { From 86ad432d5b153b4f69a7f65f5baacd2e44b48b4e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:16:40 +0530 Subject: [PATCH 091/211] [server] Implement file data deleted using existing table as queue --- server/pkg/controller/filedata/delete.go | 146 ++++++++++++++++------- server/pkg/repo/filedata/repository.go | 37 ++++++ 2 files changed, 142 insertions(+), 41 deletions(-) diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go index 799844d06b..d6d07cd24d 100644 --- a/server/pkg/controller/filedata/delete.go +++ b/server/pkg/controller/filedata/delete.go @@ -2,96 +2,160 @@ package filedata import ( "context" + "database/sql" + "errors" "fmt" "github.com/ente-io/museum/ente/filedata" fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata" + enteTime "github.com/ente-io/museum/pkg/utils/time" "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" + "time" ) // StartDataDeletion clears associated file data from the object store func (c *Controller) StartDataDeletion() { - log.Info("Cleaning up deleted file data") - // todo: start goroutine workers to delete data + go c.startDeleteWorkers(5) +} + +func (c *Controller) startDeleteWorkers(n int) { + log.Infof("Starting %d delete workers for fileData", n) + for i := 0; i < n; i++ { + go c.delete(i) + // Stagger the workers + time.Sleep(time.Duration(2*i+1) * time.Second) + } } -func (c *Controller) DeleteFileData(fileID int64) error { - ownerID, err := c.FileRepo.GetOwnerID(fileID) +// Entry point for the delete worker (goroutine) +// +// i is an arbitrary index of the current routine. +func (c *Controller) delete(i int) { + // This is just + // + // while (true) { delete() } + // + // but with an extra sleep for a bit if nothing got deleted - both when + // something's wrong, or there's nothing to do. + for { + err := c.tryDelete() + if err != nil { + // Sleep in proportion to the (arbitrary) index to space out the + // workers further. + time.Sleep(time.Duration(i+1) * time.Minute) + } + } +} + +func (c *Controller) tryDelete() error { + newLockTime := enteTime.MicrosecondsAfterMinutes(10) + row, err := c.Repo.GetPendingSyncDataAndExtendLock(context.Background(), newLockTime, true) if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + log.Errorf("Could not fetch row for deletion: %s", err) + } return err } + err = c.DeleteFileRow(*row) + if err != nil { + log.Errorf("Could not delete file data: %s", err) + return err + } + return nil +} +func (c *Controller) DeleteFileData(fileID int64) error { rows, err := c.Repo.GetFileData(context.Background(), fileID) if err != nil { return err } for i := range rows { fileDataRow := rows[i] - ctxLogger := log.WithField("file_id", fileDataRow.DeleteFromBuckets).WithField("type", fileDataRow.Type).WithField("user_id", fileDataRow.UserID) - objectKeys := filedata.AllObjects(fileID, ownerID, fileDataRow.Type) - bucketColumnMap := make(map[string]string) - bucketColumnMap, err = getMapOfbucketItToColumn(fileDataRow) + err = c.DeleteFileRow(fileDataRow) if err != nil { - ctxLogger.WithError(err).Error("Failed to get bucketColumnMap") return err } - // Delete objects and remove buckets - for bucketID, columnName := range bucketColumnMap { - for _, objectKey := range objectKeys { - err := c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKey, bucketID) - if err != nil { - ctxLogger.WithError(err).WithFields(logrus.Fields{ - "bucketID": bucketID, - "column": columnName, - "objectKey": objectKey, - }).Error("Failed to delete object from datacenter") - return err - } - } - dbErr := c.Repo.RemoveBucket(fileDataRow, bucketID, columnName) - if dbErr != nil { - ctxLogger.WithError(dbErr).WithFields(logrus.Fields{ - "bucketID": bucketID, - "column": columnName, - }).Error("Failed to remove bucket from db") - return dbErr + } + return nil +} - } - } - // Delete from Latest bucket - for k := range objectKeys { - err = c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKeys[k], fileDataRow.LatestBucket) +func (c *Controller) DeleteFileRow(fileDataRow filedata.Row) error { + if !fileDataRow.IsDeleted { + return fmt.Errorf("file %d is not marked as deleted", fileDataRow.FileID) + } + fileID := fileDataRow.FileID + ownerID, err := c.FileRepo.GetOwnerID(fileID) + if err != nil { + return err + } + if fileDataRow.UserID != ownerID { + // this should never happen + panic(fmt.Sprintf("file %d does not belong to user %d", fileID, ownerID)) + } + ctxLogger := log.WithField("file_id", fileDataRow.DeleteFromBuckets).WithField("type", fileDataRow.Type).WithField("user_id", fileDataRow.UserID) + objectKeys := filedata.AllObjects(fileID, ownerID, fileDataRow.Type) + bucketColumnMap := make(map[string]string) + bucketColumnMap, err = getMapOfBucketItToColumn(fileDataRow) + if err != nil { + ctxLogger.WithError(err).Error("Failed to get bucketColumnMap") + return err + } + // Delete objects and remove buckets + for bucketID, columnName := range bucketColumnMap { + for _, objectKey := range objectKeys { + err := c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKey, bucketID) if err != nil { - ctxLogger.WithError(err).Error("Failed to delete object from datacenter") + ctxLogger.WithError(err).WithFields(logrus.Fields{ + "bucketID": bucketID, + "column": columnName, + "objectKey": objectKey, + }).Error("Failed to delete object from datacenter") return err } } - dbErr := c.Repo.DeleteFileData(context.Background(), fileDataRow) + dbErr := c.Repo.RemoveBucket(fileDataRow, bucketID, columnName) if dbErr != nil { - ctxLogger.WithError(dbErr).Error("Failed to remove from db") + ctxLogger.WithError(dbErr).WithFields(logrus.Fields{ + "bucketID": bucketID, + "column": columnName, + }).Error("Failed to remove bucket from db") + return dbErr + + } + } + // Delete from Latest bucket + for k := range objectKeys { + err = c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKeys[k], fileDataRow.LatestBucket) + if err != nil { + ctxLogger.WithError(err).Error("Failed to delete object from datacenter") return err } } + dbErr := c.Repo.DeleteFileData(context.Background(), fileDataRow) + if dbErr != nil { + ctxLogger.WithError(dbErr).Error("Failed to remove from db") + return err + } return nil } -func getMapOfbucketItToColumn(row filedata.Row) (map[string]string, error) { +func getMapOfBucketItToColumn(row filedata.Row) (map[string]string, error) { bucketColumnMap := make(map[string]string) for _, bucketID := range row.DeleteFromBuckets { if existingColumn, exists := bucketColumnMap[bucketID]; exists { - return nil, fmt.Errorf("Duplicate DeleteFromBuckets ID found: %s in column %s", bucketID, existingColumn) + return nil, fmt.Errorf("duplicate DeleteFromBuckets ID found: %s in column %s", bucketID, existingColumn) } bucketColumnMap[bucketID] = fileDataRepo.DeletionColumn } for _, bucketID := range row.ReplicatedBuckets { if existingColumn, exists := bucketColumnMap[bucketID]; exists { - return nil, fmt.Errorf("Duplicate ReplicatedBuckets ID found: %s in column %s", bucketID, existingColumn) + return nil, fmt.Errorf("duplicate ReplicatedBuckets ID found: %s in column %s", bucketID, existingColumn) } bucketColumnMap[bucketID] = fileDataRepo.ReplicationColumn } for _, bucketID := range row.InflightReplicas { if existingColumn, exists := bucketColumnMap[bucketID]; exists { - return nil, fmt.Errorf("Duplicate InFlightBucketID found: %s in column %s", bucketID, existingColumn) + return nil, fmt.Errorf("duplicate InFlightBucketID found: %s in column %s", bucketID, existingColumn) } bucketColumnMap[bucketID] = fileDataRepo.InflightRepColumn } diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go index 21208697bd..2ecabbb8b3 100644 --- a/server/pkg/repo/filedata/repository.go +++ b/server/pkg/repo/filedata/repository.go @@ -8,6 +8,7 @@ import ( "github.com/ente-io/museum/ente/filedata" "github.com/ente-io/stacktrace" "github.com/lib/pq" + "time" ) // Repository defines the methods for inserting, updating, and retrieving file data. @@ -157,6 +158,42 @@ func (r *Repository) MoveBetweenBuckets(row filedata.Row, bucketID string, sourc return nil } +// GetPendingSyncDataAndExtendLock in a transaction gets single file data row that has been deleted and pending sync is true and sync_lock_till is less than now_utc_micro_seconds() and extends the lock till newSyncLockTime +// This is used to lock the file data row for deletion and extend +func (r *Repository) GetPendingSyncDataAndExtendLock(ctx context.Context, newSyncLockTime int64, forDeletion bool) (*filedata.Row, error) { + // ensure newSyncLockTime is in the future + if newSyncLockTime < time.Now().Add(5*time.Minute).UnixMicro() { + return nil, stacktrace.NewError("newSyncLockTime should be at least 5min in the future") + } + tx, err := r.DB.BeginTx(ctx, nil) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + defer tx.Rollback() + row := tx.QueryRow(`SELECT file_id, user_id, data_type, size, latest_bucket, replicated_buckets, delete_from_buckets, inflight_rep_buckets, pending_sync, is_deleted, sync_locked_till, created_at, updated_at + FROM file_data + where pending_sync = true and is_deleted = $1 and sync_locked_till < now_utc_micro_seconds() + LIMIT 1 + FOR UPDATE SKIP LOCKED`, forDeletion) + var fileData filedata.Row + err = row.Scan(&fileData.FileID, &fileData.UserID, &fileData.Type, &fileData.Size, &fileData.LatestBucket, pq.Array(&fileData.ReplicatedBuckets), pq.Array(&fileData.DeleteFromBuckets), pq.Array(&fileData.InflightReplicas), &fileData.PendingSync, &fileData.IsDeleted, &fileData.SyncLockedTill, &fileData.CreatedAt, &fileData.UpdatedAt) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + if fileData.SyncLockedTill > newSyncLockTime { + return nil, stacktrace.NewError(fmt.Sprintf("newSyncLockTime (%d) is less than existing SyncLockedTill(%d), newSync", newSyncLockTime, fileData.SyncLockedTill)) + } + _, err = tx.Exec(`UPDATE file_data SET sync_locked_till = $1 WHERE file_id = $2 AND data_type = $3 AND user_id = $4`, newSyncLockTime, fileData.FileID, string(fileData.Type), fileData.UserID) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + err = tx.Commit() + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &fileData, nil +} + func (r *Repository) DeleteFileData(ctx context.Context, row filedata.Row) error { query := ` DELETE FROM file_data From 58e55a7a00cd9567180101dc6658893dc16616b6 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:19:50 +0530 Subject: [PATCH 092/211] [server] Clean up --- server/pkg/controller/filedata/delete.go | 14 -------------- server/pkg/repo/filedata/repository.go | 5 +++-- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go index d6d07cd24d..05da540d5b 100644 --- a/server/pkg/controller/filedata/delete.go +++ b/server/pkg/controller/filedata/delete.go @@ -64,20 +64,6 @@ func (c *Controller) tryDelete() error { } return nil } -func (c *Controller) DeleteFileData(fileID int64) error { - rows, err := c.Repo.GetFileData(context.Background(), fileID) - if err != nil { - return err - } - for i := range rows { - fileDataRow := rows[i] - err = c.DeleteFileRow(fileDataRow) - if err != nil { - return err - } - } - return nil -} func (c *Controller) DeleteFileRow(fileDataRow filedata.Row) error { if !fileDataRow.IsDeleted { diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go index 2ecabbb8b3..bdf184591e 100644 --- a/server/pkg/repo/filedata/repository.go +++ b/server/pkg/repo/filedata/repository.go @@ -196,8 +196,9 @@ func (r *Repository) GetPendingSyncDataAndExtendLock(ctx context.Context, newSyn func (r *Repository) DeleteFileData(ctx context.Context, row filedata.Row) error { query := ` -DELETE FROM file_data -WHERE file_id = $1 AND data_type = $2 AND latest_bucket = $3 AND user_id = $4 AND replicated_buckets = ARRAY[]::s3region[] AND delete_from_buckets = ARRAY[]::s3region[]` + DELETE FROM file_data + WHERE file_id = $1 AND data_type = $2 AND latest_bucket = $3 AND user_id = $4 + AND replicated_buckets = ARRAY[]::s3region[] AND delete_from_buckets = ARRAY[]::s3region[] and inflight_rep_buckets = ARRAY[]::s3region[] and is_deleted=True` res, err := r.DB.ExecContext(ctx, query, row.FileID, string(row.Type), row.LatestBucket, row.UserID) if err != nil { return stacktrace.Propagate(err, "") From 4920ecf643547461b05cbbc346f78896a5ba37cf Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Thu, 8 Aug 2024 14:44:42 +0530 Subject: [PATCH 093/211] rename --- server/pkg/controller/filedata/delete.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go index 05da540d5b..782f30d370 100644 --- a/server/pkg/controller/filedata/delete.go +++ b/server/pkg/controller/filedata/delete.go @@ -57,7 +57,7 @@ func (c *Controller) tryDelete() error { } return err } - err = c.DeleteFileRow(*row) + err = c.deleteFileRow(*row) if err != nil { log.Errorf("Could not delete file data: %s", err) return err @@ -65,7 +65,7 @@ func (c *Controller) tryDelete() error { return nil } -func (c *Controller) DeleteFileRow(fileDataRow filedata.Row) error { +func (c *Controller) deleteFileRow(fileDataRow filedata.Row) error { if !fileDataRow.IsDeleted { return fmt.Errorf("file %d is not marked as deleted", fileDataRow.FileID) } From 7dcfe12d1d6944219d68fed42169e4a9e773adad Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Thu, 8 Aug 2024 20:17:16 +0530 Subject: [PATCH 094/211] [web] Fix referrer policy Ref: https://web.dev/articles/referrer-best-practices --- web/apps/auth/public/_headers | 1 - web/apps/photos/public/_headers | 1 - web/packages/base/components/Head.tsx | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/web/apps/auth/public/_headers b/web/apps/auth/public/_headers index 72dc5bb5ce..13dd2408e7 100644 --- a/web/apps/auth/public/_headers +++ b/web/apps/auth/public/_headers @@ -5,6 +5,5 @@ X-Download-Options: noopen X-Frame-Options: deny X-XSS-Protection: 1; mode=block - Referrer-Policy: same-origin Content-Security-Policy-Report-Only: default-src 'self'; img-src 'self' blob: data:; media-src 'self' blob:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self' 'unsafe-eval' blob:; manifest-src 'self'; child-src 'self' blob:; object-src 'none'; connect-src 'self' https://*.ente.io data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com https://ente-prod-v3.s3.eu-central-2.wasabisys.com/ ; base-uri 'self'; frame-ancestors 'none'; form-action 'none'; report-uri https://csp-reporter.ente.io; report-to https://csp-reporter.ente.io; diff --git a/web/apps/photos/public/_headers b/web/apps/photos/public/_headers index 72dc5bb5ce..13dd2408e7 100644 --- a/web/apps/photos/public/_headers +++ b/web/apps/photos/public/_headers @@ -5,6 +5,5 @@ X-Download-Options: noopen X-Frame-Options: deny X-XSS-Protection: 1; mode=block - Referrer-Policy: same-origin Content-Security-Policy-Report-Only: default-src 'self'; img-src 'self' blob: data:; media-src 'self' blob:; style-src 'self' 'unsafe-inline'; font-src 'self'; script-src 'self' 'unsafe-eval' blob:; manifest-src 'self'; child-src 'self' blob:; object-src 'none'; connect-src 'self' https://*.ente.io data: blob: https://ente-prod-eu.s3.eu-central-003.backblazeb2.com https://ente-prod-v3.s3.eu-central-2.wasabisys.com/ ; base-uri 'self'; frame-ancestors 'none'; form-action 'none'; report-uri https://csp-reporter.ente.io; report-to https://csp-reporter.ente.io; diff --git a/web/packages/base/components/Head.tsx b/web/packages/base/components/Head.tsx index d756a9b09b..0373bd3aea 100644 --- a/web/packages/base/components/Head.tsx +++ b/web/packages/base/components/Head.tsx @@ -24,6 +24,7 @@ export const CustomHead: React.FC = ({ title }) => { name="viewport" content="width=device-width, initial-scale=1" /> + ); }; From 5925dfb3fc1188f87638149f57d58414276737fe Mon Sep 17 00:00:00 2001 From: Aman Raj Singh Mourya Date: Thu, 8 Aug 2024 21:41:46 +0530 Subject: [PATCH 095/211] [mob][photos] Minor fixes --- mobile/lib/events/file_swipe_lock_event.dart | 7 ------- mobile/lib/events/guest_view_event.dart | 7 +++++++ .../file_selection_actions_widget.dart | 4 ++-- mobile/lib/ui/viewer/file/detail_page.dart | 21 ++++++++----------- mobile/lib/ui/viewer/file/file_app_bar.dart | 15 +++++++------ .../lib/ui/viewer/file/file_bottom_bar.dart | 11 +++++----- mobile/lib/ui/viewer/file/video_widget.dart | 11 +++++----- .../lib/ui/viewer/file/video_widget_new.dart | 11 +++++----- mobile/lib/ui/viewer/file/zoomable_image.dart | 17 ++++----------- .../viewer/file/zoomable_live_image_new.dart | 18 ++++++++++++++-- 10 files changed, 60 insertions(+), 62 deletions(-) delete mode 100644 mobile/lib/events/file_swipe_lock_event.dart create mode 100644 mobile/lib/events/guest_view_event.dart diff --git a/mobile/lib/events/file_swipe_lock_event.dart b/mobile/lib/events/file_swipe_lock_event.dart deleted file mode 100644 index 48aed9bcf5..0000000000 --- a/mobile/lib/events/file_swipe_lock_event.dart +++ /dev/null @@ -1,7 +0,0 @@ -import "package:photos/events/event.dart"; - -class FileSwipeLockEvent extends Event { - final bool isGuestView; - final bool swipeLocked; - FileSwipeLockEvent(this.isGuestView, this.swipeLocked); -} diff --git a/mobile/lib/events/guest_view_event.dart b/mobile/lib/events/guest_view_event.dart new file mode 100644 index 0000000000..6c74f1f4ab --- /dev/null +++ b/mobile/lib/events/guest_view_event.dart @@ -0,0 +1,7 @@ +import "package:photos/events/event.dart"; + +class GuestViewEvent extends Event { + final bool isGuestView; + final bool swipeLocked; + GuestViewEvent(this.isGuestView, this.swipeLocked); +} diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index b37b17f671..148d97f5e6 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -8,7 +8,7 @@ import "package:logging/logging.dart"; import "package:modal_bottom_sheet/modal_bottom_sheet.dart"; import 'package:photos/core/configuration.dart'; import "package:photos/core/event_bus.dart"; -import "package:photos/events/file_swipe_lock_event.dart"; +import "package:photos/events/guest_view_event.dart"; import "package:photos/events/people_changed_event.dart"; import "package:photos/face/model/person.dart"; import "package:photos/generated/l10n.dart"; @@ -578,7 +578,7 @@ class _FileSelectionActionsWidgetState ); routeToPage(context, page, forceCustomPageRoute: true).ignore(); WidgetsBinding.instance.addPostFrameCallback((_) { - Bus.instance.fire(FileSwipeLockEvent(true, false)); + Bus.instance.fire(GuestViewEvent(true, false)); }); widget.selectedFiles.clearAll(); } diff --git a/mobile/lib/ui/viewer/file/detail_page.dart b/mobile/lib/ui/viewer/file/detail_page.dart index bcf3d0540c..64e1e73ac1 100644 --- a/mobile/lib/ui/viewer/file/detail_page.dart +++ b/mobile/lib/ui/viewer/file/detail_page.dart @@ -10,7 +10,7 @@ import 'package:photos/core/configuration.dart'; import 'package:photos/core/constants.dart'; import 'package:photos/core/errors.dart'; import "package:photos/core/event_bus.dart"; -import "package:photos/events/file_swipe_lock_event.dart"; +import "package:photos/events/guest_view_event.dart"; import "package:photos/generated/l10n.dart"; import 'package:photos/models/file/file.dart'; import "package:photos/models/file/file_type.dart"; @@ -88,8 +88,7 @@ class _DetailPageState extends State { bool _isFirstOpened = true; bool isGuestView = false; bool swipeLocked = false; - late final StreamSubscription - _fileSwipeLockEventSubscription; + late final StreamSubscription _guestViewEventSubscription; @override void initState() { @@ -100,8 +99,8 @@ class _DetailPageState extends State { _selectedIndexNotifier.value = widget.config.selectedIndex; _preloadEntries(); _pageController = PageController(initialPage: _selectedIndexNotifier.value); - _fileSwipeLockEventSubscription = - Bus.instance.on().listen((event) { + _guestViewEventSubscription = + Bus.instance.on().listen((event) { setState(() { isGuestView = event.isGuestView; swipeLocked = event.swipeLocked; @@ -111,7 +110,7 @@ class _DetailPageState extends State { @override void dispose() { - _fileSwipeLockEventSubscription.cancel(); + _guestViewEventSubscription.cancel(); _pageController.dispose(); _enableFullScreenNotifier.dispose(); _selectedIndexNotifier.dispose(); @@ -145,7 +144,7 @@ class _DetailPageState extends State { if (isGuestView) { final authenticated = await _requestAuthentication(); if (authenticated) { - Bus.instance.fire(FileSwipeLockEvent(false, false)); + Bus.instance.fire(GuestViewEvent(false, false)); } } }, @@ -237,13 +236,11 @@ class _DetailPageState extends State { } else { _selectedIndexNotifier.value = index; } - Bus.instance.fire(FileSwipeLockEvent(isGuestView, swipeLocked)); + Bus.instance.fire(GuestViewEvent(isGuestView, swipeLocked)); _preloadEntries(); }, - physics: _shouldDisableScroll || isGuestView - ? swipeLocked - ? const NeverScrollableScrollPhysics() - : const FastScrollPhysics(speedFactor: 4.0) + physics: _shouldDisableScroll || swipeLocked + ? const NeverScrollableScrollPhysics() : const FastScrollPhysics(speedFactor: 4.0), controller: _pageController, itemCount: _files!.length, diff --git a/mobile/lib/ui/viewer/file/file_app_bar.dart b/mobile/lib/ui/viewer/file/file_app_bar.dart index f6d83169c6..28ba653e2c 100644 --- a/mobile/lib/ui/viewer/file/file_app_bar.dart +++ b/mobile/lib/ui/viewer/file/file_app_bar.dart @@ -6,7 +6,7 @@ import "package:local_auth/local_auth.dart"; import 'package:logging/logging.dart'; import 'package:media_extension/media_extension.dart'; import "package:photos/core/event_bus.dart"; -import "package:photos/events/file_swipe_lock_event.dart"; +import "package:photos/events/guest_view_event.dart"; import "package:photos/generated/l10n.dart"; import "package:photos/l10n/l10n.dart"; import "package:photos/models/file/extensions/file_props.dart"; @@ -51,8 +51,7 @@ class FileAppBar extends StatefulWidget { class FileAppBarState extends State { final _logger = Logger("FadingAppBar"); final List _actions = []; - late final StreamSubscription - _fileSwipeLockEventSubscription; + late final StreamSubscription _guestViewEventSubscription; bool isGuestView = false; @override @@ -66,8 +65,8 @@ class FileAppBarState extends State { @override void initState() { super.initState(); - _fileSwipeLockEventSubscription = - Bus.instance.on().listen((event) { + _guestViewEventSubscription = + Bus.instance.on().listen((event) { setState(() { isGuestView = event.isGuestView; }); @@ -76,7 +75,7 @@ class FileAppBarState extends State { @override void dispose() { - _fileSwipeLockEventSubscription.cancel(); + _guestViewEventSubscription.cancel(); super.dispose(); } @@ -415,7 +414,7 @@ class FileAppBarState extends State { Future _onTapGuestView() async { if (await LocalAuthentication().isDeviceSupported()) { - Bus.instance.fire(FileSwipeLockEvent(!isGuestView, true)); + Bus.instance.fire(GuestViewEvent(true, true)); } else { await showErrorDialog( context, @@ -432,7 +431,7 @@ class FileAppBarState extends State { "Please authenticate to view more photos and videos.", ); if (hasAuthenticated) { - Bus.instance.fire(FileSwipeLockEvent(false, false)); + Bus.instance.fire(GuestViewEvent(false, false)); } } } diff --git a/mobile/lib/ui/viewer/file/file_bottom_bar.dart b/mobile/lib/ui/viewer/file/file_bottom_bar.dart index a569656b29..2c9781cbc5 100644 --- a/mobile/lib/ui/viewer/file/file_bottom_bar.dart +++ b/mobile/lib/ui/viewer/file/file_bottom_bar.dart @@ -5,7 +5,7 @@ import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import "package:logging/logging.dart"; import "package:photos/core/event_bus.dart"; -import "package:photos/events/file_swipe_lock_event.dart"; +import "package:photos/events/guest_view_event.dart"; import "package:photos/generated/l10n.dart"; import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; @@ -48,16 +48,15 @@ class FileBottomBar extends StatefulWidget { class FileBottomBarState extends State { final GlobalKey shareButtonKey = GlobalKey(); bool isGuestView = false; - late final StreamSubscription - _fileSwipeLockEventSubscription; + late final StreamSubscription _guestViewEventSubscription; bool isPanorama = false; int? lastFileGenID; @override void initState() { super.initState(); - _fileSwipeLockEventSubscription = - Bus.instance.on().listen((event) { + _guestViewEventSubscription = + Bus.instance.on().listen((event) { setState(() { isGuestView = event.isGuestView; }); @@ -66,7 +65,7 @@ class FileBottomBarState extends State { @override void dispose() { - _fileSwipeLockEventSubscription.cancel(); + _guestViewEventSubscription.cancel(); super.dispose(); } diff --git a/mobile/lib/ui/viewer/file/video_widget.dart b/mobile/lib/ui/viewer/file/video_widget.dart index b9d06ecaff..d700690edb 100644 --- a/mobile/lib/ui/viewer/file/video_widget.dart +++ b/mobile/lib/ui/viewer/file/video_widget.dart @@ -7,7 +7,7 @@ import 'package:flutter/material.dart'; import 'package:logging/logging.dart'; import 'package:photos/core/constants.dart'; import "package:photos/core/event_bus.dart"; -import "package:photos/events/file_swipe_lock_event.dart"; +import "package:photos/events/guest_view_event.dart"; import "package:photos/generated/l10n.dart"; import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; @@ -49,8 +49,7 @@ class _VideoWidgetState extends State { bool _isPlaying = false; final EnteWakeLock _wakeLock = EnteWakeLock(); bool isGuestView = false; - late final StreamSubscription - _fileSwipeLockEventSubscription; + late final StreamSubscription _guestViewEventSubscription; @override void initState() { @@ -80,8 +79,8 @@ class _VideoWidgetState extends State { } }); } - _fileSwipeLockEventSubscription = - Bus.instance.on().listen((event) { + _guestViewEventSubscription = + Bus.instance.on().listen((event) { setState(() { isGuestView = event.isGuestView; }); @@ -132,7 +131,7 @@ class _VideoWidgetState extends State { @override void dispose() { - _fileSwipeLockEventSubscription.cancel(); + _guestViewEventSubscription.cancel(); removeCallBack(widget.file); _videoPlayerController?.dispose(); _chewieController?.dispose(); diff --git a/mobile/lib/ui/viewer/file/video_widget_new.dart b/mobile/lib/ui/viewer/file/video_widget_new.dart index f1a91b77ff..274a5fce48 100644 --- a/mobile/lib/ui/viewer/file/video_widget_new.dart +++ b/mobile/lib/ui/viewer/file/video_widget_new.dart @@ -8,7 +8,7 @@ import "package:media_kit/media_kit.dart"; import "package:media_kit_video/media_kit_video.dart"; import "package:photos/core/constants.dart"; import "package:photos/core/event_bus.dart"; -import "package:photos/events/file_swipe_lock_event.dart"; +import "package:photos/events/guest_view_event.dart"; import "package:photos/events/pause_video_event.dart"; import "package:photos/generated/l10n.dart"; import "package:photos/models/file/extensions/file_props.dart"; @@ -48,8 +48,7 @@ class _VideoWidgetNewState extends State bool _isAppInFG = true; late StreamSubscription pauseVideoSubscription; bool isGuestView = false; - late final StreamSubscription - _fileSwipeLockEventSubscription; + late final StreamSubscription _guestViewEventSubscription; @override void initState() { @@ -94,8 +93,8 @@ class _VideoWidgetNewState extends State pauseVideoSubscription = Bus.instance.on().listen((event) { player.pause(); }); - _fileSwipeLockEventSubscription = - Bus.instance.on().listen((event) { + _guestViewEventSubscription = + Bus.instance.on().listen((event) { setState(() { isGuestView = event.isGuestView; }); @@ -113,7 +112,7 @@ class _VideoWidgetNewState extends State @override void dispose() { - _fileSwipeLockEventSubscription.cancel(); + _guestViewEventSubscription.cancel(); pauseVideoSubscription.cancel(); removeCallBack(widget.file); _progressNotifier.dispose(); diff --git a/mobile/lib/ui/viewer/file/zoomable_image.dart b/mobile/lib/ui/viewer/file/zoomable_image.dart index 64a7b2e0e5..0743cba757 100644 --- a/mobile/lib/ui/viewer/file/zoomable_image.dart +++ b/mobile/lib/ui/viewer/file/zoomable_image.dart @@ -9,8 +9,7 @@ import 'package:photos/core/cache/thumbnail_in_memory_cache.dart'; import 'package:photos/core/constants.dart'; import 'package:photos/core/event_bus.dart'; import 'package:photos/db/files_db.dart'; -import "package:photos/events/file_swipe_lock_event.dart"; -import 'package:photos/events/files_updated_event.dart'; +import "package:photos/events/files_updated_event.dart"; import 'package:photos/events/local_photos_updated_event.dart'; import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; @@ -26,6 +25,7 @@ class ZoomableImage extends StatefulWidget { final String? tagPrefix; final Decoration? backgroundDecoration; final bool shouldCover; + final bool isGuestView; const ZoomableImage( this.photo, { @@ -34,6 +34,7 @@ class ZoomableImage extends StatefulWidget { required this.tagPrefix, this.backgroundDecoration, this.shouldCover = false, + this.isGuestView = false, }); @override @@ -54,9 +55,6 @@ class _ZoomableImageState extends State { bool _isZooming = false; PhotoViewController _photoViewController = PhotoViewController(); final _scaleStateController = PhotoViewScaleStateController(); - bool isGuestView = false; - late final StreamSubscription - _fileSwipeLockEventSubscription; @override void initState() { @@ -72,17 +70,10 @@ class _ZoomableImageState extends State { debugPrint("isZooming = $_isZooming, currentState $value"); // _logger.info('is reakky zooming $_isZooming with state $value'); }; - _fileSwipeLockEventSubscription = - Bus.instance.on().listen((event) { - setState(() { - isGuestView = event.isGuestView; - }); - }); } @override void dispose() { - _fileSwipeLockEventSubscription.cancel(); _photoViewController.dispose(); _scaleStateController.dispose(); super.dispose(); @@ -159,7 +150,7 @@ class _ZoomableImageState extends State { } final GestureDragUpdateCallback? verticalDragCallback = - _isZooming || isGuestView + _isZooming || widget.isGuestView ? null : (d) => { if (!_isZooming) diff --git a/mobile/lib/ui/viewer/file/zoomable_live_image_new.dart b/mobile/lib/ui/viewer/file/zoomable_live_image_new.dart index 598ae60e02..036a3a62f0 100644 --- a/mobile/lib/ui/viewer/file/zoomable_live_image_new.dart +++ b/mobile/lib/ui/viewer/file/zoomable_live_image_new.dart @@ -1,3 +1,4 @@ +import "dart:async"; import "dart:io"; import 'package:flutter/material.dart'; @@ -5,6 +6,8 @@ import 'package:logging/logging.dart'; import "package:media_kit/media_kit.dart"; import "package:media_kit_video/media_kit_video.dart"; import 'package:motion_photos/motion_photos.dart'; +import "package:photos/core/event_bus.dart"; +import "package:photos/events/guest_view_event.dart"; import "package:photos/generated/l10n.dart"; import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; @@ -23,11 +26,11 @@ class ZoomableLiveImageNew extends StatefulWidget { const ZoomableLiveImageNew( this.enteFile, { - Key? key, + super.key, this.shouldDisableScroll, required this.tagPrefix, this.backgroundDecoration, - }) : super(key: key); + }); @override State createState() => _ZoomableLiveImageNewState(); @@ -43,6 +46,9 @@ class _ZoomableLiveImageNewState extends State late final _player = Player(); VideoController? _videoController; + bool isGuestView = false; + late final StreamSubscription _guestViewEventSubscription; + @override void initState() { _enteFile = widget.enteFile; @@ -52,6 +58,12 @@ class _ZoomableLiveImageNewState extends State if (_enteFile.isLivePhoto && _enteFile.isUploaded) { LocalFileUpdateService.instance.checkLivePhoto(_enteFile).ignore(); } + _guestViewEventSubscription = + Bus.instance.on().listen((event) { + setState(() { + isGuestView = event.isGuestView; + }); + }); super.initState(); } @@ -83,6 +95,7 @@ class _ZoomableLiveImageNewState extends State tagPrefix: widget.tagPrefix, shouldDisableScroll: widget.shouldDisableScroll, backgroundDecoration: widget.backgroundDecoration, + isGuestView: isGuestView, ); } return GestureDetector( @@ -98,6 +111,7 @@ class _ZoomableLiveImageNewState extends State _videoController!.player.stop(); _videoController!.player.dispose(); } + _guestViewEventSubscription.cancel(); super.dispose(); } From fb2c17c510fbf0ed8f4799c52e09a87caa3ae8c2 Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Fri, 9 Aug 2024 02:32:14 +0530 Subject: [PATCH 096/211] fix: update panorama package to support sensor tweaks --- .../viewer/file/panorama_viewer_screen.dart | 4 +-- mobile/pubspec.lock | 27 ++++++++++--------- mobile/pubspec.yaml | 5 ++-- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart b/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart index b54b63879c..c9437b5ab6 100644 --- a/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart +++ b/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart @@ -4,7 +4,7 @@ import "dart:io"; import "package:flutter/material.dart"; import "package:flutter/services.dart"; import "package:motion_photos/src/xmp_extractor.dart"; -import "package:panorama_viewer/panorama_viewer.dart"; +import "package:panorama/panorama.dart"; import "package:photos/generated/l10n.dart"; class PanoramaViewerScreen extends StatefulWidget { @@ -123,7 +123,7 @@ class _PanoramaViewerScreenState extends State { return Scaffold( body: Stack( children: [ - PanoramaViewer( + Panorama( onTap: (_, __, ___) { setState(() { if (isVisible) { diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index e9cec130c2..5adbd5c31c 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -387,14 +387,6 @@ packages: url: "https://pub.dev" source: hosted version: "0.7.10" - dchs_motion_sensors: - dependency: transitive - description: - name: dchs_motion_sensors - sha256: "0f1cac6cd876fad2afff47405a963913f39a67221cbfc4f00aa04da2845bab0a" - url: "https://pub.dev" - source: hosted - version: "1.1.0" defer_pointer: dependency: "direct main" description: @@ -1581,6 +1573,15 @@ packages: url: "https://github.com/ente-io/motion_photo.git" source: git version: "0.0.6" + motion_sensors: + dependency: transitive + description: + path: "." + ref: aves + resolved-ref: "4b11d59f4bda152627f701070272f657f8358e67" + url: "https://github.com/deckerst/aves_panorama_motion_sensors.git" + source: git + version: "0.1.0" motionphoto: dependency: "direct main" description: @@ -1687,15 +1688,15 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.0" - panorama_viewer: + panorama: dependency: "direct main" description: path: "." - ref: HEAD - resolved-ref: "26ad55b2aa29dde14d640d6f1e17ce132a87910a" - url: "https://github.com/prateekmedia/panorama_viewer.git" + ref: blur + resolved-ref: "362f0295ba4ed42486577c9f9724c462107080f8" + url: "https://github.com/ente-io/panorama_blur.git" source: git - version: "1.0.5" + version: "0.4.1" password_strength: dependency: "direct main" description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index e090092713..d3a122d259 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -131,9 +131,10 @@ dependencies: open_mail_app: ^0.4.5 package_info_plus: ^4.1.0 page_transition: ^2.0.2 - panorama_viewer: + panorama: git: - url: https://github.com/ente-io/panorama_viewer.git + url: https://github.com/ente-io/panorama_blur.git + ref: blur password_strength: ^0.2.0 path: #dart path_provider: ^2.1.1 From 10d6caa4e1effdca761c12a8886714ecb860a3b7 Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Fri, 9 Aug 2024 02:40:03 +0530 Subject: [PATCH 097/211] fix: review changes --- mobile/lib/utils/panorama_util.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mobile/lib/utils/panorama_util.dart b/mobile/lib/utils/panorama_util.dart index c002873185..909801a546 100644 --- a/mobile/lib/utils/panorama_util.dart +++ b/mobile/lib/utils/panorama_util.dart @@ -46,7 +46,7 @@ Future guardedCheckPanorama(EnteFile file) async { final result = await checkIfPanorama(file); // Update the metadata if it is not updated - if (file.isPanorama() == null && file.canEditMetaInfo) { + if (file.canEditMetaInfo && file.isPanorama() == null) { int? mediaType = file.pubMagicMetadata?.mediaType; mediaType ??= 0; From 8db29a25a45d1fe93788afeb4505217eecb22993 Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Fri, 9 Aug 2024 02:41:44 +0530 Subject: [PATCH 098/211] fix: review changes --- mobile/lib/models/file/file.dart | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/mobile/lib/models/file/file.dart b/mobile/lib/models/file/file.dart index b4276878ce..40d4a388af 100644 --- a/mobile/lib/models/file/file.dart +++ b/mobile/lib/models/file/file.dart @@ -183,13 +183,15 @@ class EnteFile { } mediaUploadData.isPanorama = checkPanoramaFromEXIF(null, exifData); - try { - final xmpData = XMPExtractor() - .extract(mediaUploadData.sourceFile!.readAsBytesSync()); - mediaUploadData.isPanorama = checkPanoramaFromXMP(xmpData); - } catch (_) {} - - mediaUploadData.isPanorama ??= false; + if (mediaUploadData.isPanorama != true) { + try { + final xmpData = XMPExtractor() + .extract(mediaUploadData.sourceFile!.readAsBytesSync()); + mediaUploadData.isPanorama = checkPanoramaFromXMP(xmpData); + } catch (_) {} + + mediaUploadData.isPanorama ??= false; + } } if (Platform.isAndroid) { //Fix for missing location data in lower android versions. From 685680c6daec87c96dd9ff7965a2efe73d5edc98 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 09:44:26 +0530 Subject: [PATCH 099/211] Integrate --- web/apps/photos/src/services/searchService.ts | 2 ++ .../new/photos/services/ml/cluster-new.ts | 10 ++++---- web/packages/new/photos/services/ml/index.ts | 24 ++++++++++++++++--- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 750a1fb186..8488462aba 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -6,6 +6,7 @@ import { isMLEnabled, isMLSupported, mlStatusSnapshot, + wipCluster, } from "@/new/photos/services/ml"; import type { Person } from "@/new/photos/services/ml/people"; import { EnteFile } from "@/new/photos/types/file"; @@ -415,6 +416,7 @@ function convertSuggestionToSearchQuery(option: Suggestion): Search { async function getAllPeople(limit: number = undefined) { let people: Array = []; // await mlIDbStorage.getAllPeople(); + people = await wipCluster(); // await mlPeopleStore.iterate((person) => { // people.push(person); // }); diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index c5e3514a67..bee7c2fee8 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -5,11 +5,11 @@ import type { FaceIndex } from "./face"; import { dotProduct } from "./math"; /** - * A cluster is an set of faces. + * A face cluster is an set of faces. * * Each cluster has an id so that a Person (a set of clusters) can refer to it. */ -export interface Cluster { +export interface FaceCluster { /** * A randomly generated ID to uniquely identify this cluster. */ @@ -31,7 +31,7 @@ export interface Cluster { * * For ease of transportation, the Person entity on remote looks like * - * { name, clusters: { cluster_id, face_ids }} + * { name, clusters: [{ clusterID, faceIDs }] } * * That is, it has the clusters embedded within itself. */ @@ -78,7 +78,7 @@ export const clusterFaces = (faceIndexes: FaceIndex[]) => { const faces = [...faceIDAndEmbeddings(faceIndexes)]; - const clusters: Cluster[] = []; + const clusters: FaceCluster[] = []; const clusterIndexByFaceID = new Map(); for (const [i, { faceID, embedding }] of faces.entries()) { let j = 0; @@ -118,7 +118,7 @@ export const clusterFaces = (faceIndexes: FaceIndex[]) => { `Clustered ${faces.length} faces into ${clusters.length} clusters (${Date.now() - t} ms)`, ); - return undefined; + return clusters; }; /** diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 55e482b464..22e86774ac 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -25,6 +25,7 @@ import { faceIndexes, indexableAndIndexedCounts, } from "./db"; +import type { Person } from "./people"; import { MLWorker } from "./worker"; import type { CLIPMatches } from "./worker-types"; @@ -262,8 +263,6 @@ const mlSync = async () => { triggerStatusUpdate(); if (_isMLEnabled) void worker().then((w) => w.sync()); - // TODO-ML - if (_isMLEnabled) void wipCluster(); }; /** @@ -288,6 +287,8 @@ export const indexNewUpload = (enteFile: EnteFile, uploadItem: UploadItem) => { void worker().then((w) => w.onUpload(enteFile, uploadItem)); }; +let last: Person[] = []; + /** * WIP! Don't enable, dragon eggs are hatching here. */ @@ -295,7 +296,24 @@ export const wipCluster = async () => { if (!isDevBuild || !(await isInternalUser())) return; if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return; - clusterFaces(await faceIndexes()); + if (last.length) return last; + + const clusters = clusterFaces(await faceIndexes()); + + const people: Person[] = []; // await mlIDbStorage.getAllPeople(); + for (const cluster of clusters) { + people.push({ + id: Math.random(), //cluster.id, + name: "test", + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + files: cluster.faceIDs.map((s) => parseInt(s.split("_")[0]!)), + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + displayFaceId: cluster.faceIDs[0]!, + }); + } + + last = people; + return people; }; export type MLStatus = From 292a8eb00f088f6e1f4543528c96f8ec1984c994 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 10:31:05 +0530 Subject: [PATCH 100/211] Group state --- web/packages/new/photos/services/ml/index.ts | 117 +++++++++++-------- 1 file changed, 68 insertions(+), 49 deletions(-) diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 22e86774ac..03e39343fb 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -30,39 +30,53 @@ import { MLWorker } from "./worker"; import type { CLIPMatches } from "./worker-types"; /** - * In-memory flag that tracks if ML is enabled. + * Internal state of the ML subsystem. * - * - On app start, this is read from local storage during {@link initML}. + * This are essentially cached values used by the functions of this module. * - * - It gets updated when we sync with remote (so if the user enables/disables - * ML on a different device, this local value will also become true/false). - * - * - It gets updated when the user enables/disables ML on this device. - * - * - It is cleared in {@link logoutML}. - */ -let _isMLEnabled = false; - -/** Cached instance of the {@link ComlinkWorker} that wraps our web worker. */ -let _comlinkWorker: Promise> | undefined; - -/** - * Subscriptions to {@link MLStatus}. - * - * See {@link mlStatusSubscribe}. - */ -let _mlStatusListeners: (() => void)[] = []; - -/** - * Snapshot of {@link MLStatus}. - * - * See {@link mlStatusSnapshot}. + * This should be cleared on logout. */ -let _mlStatusSnapshot: MLStatus | undefined; +class MLState { + /** + * In-memory flag that tracks if ML is enabled. + * + * - On app start, this is read from local storage during {@link initML}. + * + * - It gets updated when we sync with remote (so if the user enables/disables + * ML on a different device, this local value will also become true/false). + * + * - It gets updated when the user enables/disables ML on this device. + * + * - It is cleared in {@link logoutML}. + */ + isMLEnabled = false; + + /** + * Cached instance of the {@link ComlinkWorker} that wraps our web worker. + */ + comlinkWorker: Promise> | undefined; + + /** + * Subscriptions to {@link MLStatus}. + * + * See {@link mlStatusSubscribe}. + */ + mlStatusListeners: (() => void)[] = []; + + /** + * Snapshot of {@link MLStatus}. + * + * See {@link mlStatusSnapshot}. + */ + mlStatusSnapshot: MLStatus | undefined; +} + +/** State shared by the functions in this module. See {@link MLState}. */ +let _state = new MLState(); /** Lazily created, cached, instance of {@link MLWorker}. */ const worker = () => - (_comlinkWorker ??= createComlinkWorker()).then((cw) => cw.remote); + (_state.comlinkWorker ??= createComlinkWorker()).then((cw) => cw.remote); const createComlinkWorker = async () => { const electron = ensureElectron(); @@ -96,9 +110,9 @@ const createComlinkWorker = async () => { * It is also called when the user pauses or disables ML. */ export const terminateMLWorker = async () => { - if (_comlinkWorker) { - await _comlinkWorker.then((cw) => cw.terminate()); - _comlinkWorker = undefined; + if (_state.comlinkWorker) { + await _state.comlinkWorker.then((cw) => cw.terminate()); + _state.comlinkWorker = undefined; } }; @@ -150,7 +164,7 @@ export const canEnableML = async () => * Initialize the ML subsystem if the user has enabled it in preferences. */ export const initML = () => { - _isMLEnabled = isMLEnabledLocal(); + _state.isMLEnabled = isMLEnabledLocal(); }; export const logoutML = async () => { @@ -159,9 +173,7 @@ export const logoutML = async () => { // execution contexts], it gets called first in the logout sequence, and // then this function (`logoutML`) gets called at a later point in time. - _isMLEnabled = false; - _mlStatusListeners = []; - _mlStatusSnapshot = undefined; + _state = new MLState(); await clearMLDB(); }; @@ -174,7 +186,7 @@ export const logoutML = async () => { */ export const isMLEnabled = () => // Implementation note: Keep it fast, it might be called frequently. - _isMLEnabled; + _state.isMLEnabled; /** * Enable ML. @@ -184,7 +196,7 @@ export const isMLEnabled = () => export const enableML = async () => { await updateIsMLEnabledRemote(true); setIsMLEnabledLocal(true); - _isMLEnabled = true; + _state.isMLEnabled = true; setInterimScheduledStatus(); triggerStatusUpdate(); triggerMLSync(); @@ -199,7 +211,7 @@ export const enableML = async () => { export const disableML = async () => { await updateIsMLEnabledRemote(false); setIsMLEnabledLocal(false); - _isMLEnabled = false; + _state.isMLEnabled = false; await terminateMLWorker(); triggerStatusUpdate(); }; @@ -258,16 +270,18 @@ const updateIsMLEnabledRemote = (enabled: boolean) => export const triggerMLSync = () => void mlSync(); const mlSync = async () => { - _isMLEnabled = await getIsMLEnabledRemote(); - setIsMLEnabledLocal(_isMLEnabled); + _state.isMLEnabled = await getIsMLEnabledRemote(); + setIsMLEnabledLocal(_state.isMLEnabled); triggerStatusUpdate(); - if (_isMLEnabled) void worker().then((w) => w.sync()); + if (_state.isMLEnabled) void worker().then((w) => w.sync()); }; /** * Run indexing on a file which was uploaded from this client. * + * Indexing only happens if ML is enabled. + * * This function is called by the uploader when it uploads a new file from this * client, giving us the opportunity to index it live. This is only an * optimization - if we don't index it now it'll anyways get indexed later as @@ -281,7 +295,7 @@ const mlSync = async () => { * image part of the live photo that was uploaded. */ export const indexNewUpload = (enteFile: EnteFile, uploadItem: UploadItem) => { - if (!_isMLEnabled) return; + if (!isMLEnabled()) return; if (enteFile.metadata.fileType !== FileType.image) return; log.debug(() => ["ml/liveq", { enteFile, uploadItem }]); void worker().then((w) => w.onUpload(enteFile, uploadItem)); @@ -354,9 +368,11 @@ export type MLStatus = * @returns A function that can be used to clear the subscription. */ export const mlStatusSubscribe = (onChange: () => void): (() => void) => { - _mlStatusListeners.push(onChange); + _state.mlStatusListeners.push(onChange); return () => { - _mlStatusListeners = _mlStatusListeners.filter((l) => l != onChange); + _state.mlStatusListeners = _state.mlStatusListeners.filter( + (l) => l != onChange, + ); }; }; @@ -370,7 +386,7 @@ export const mlStatusSubscribe = (onChange: () => void): (() => void) => { * asynchronous tasks that are needed to get the status. */ export const mlStatusSnapshot = (): MLStatus | undefined => { - const result = _mlStatusSnapshot; + const result = _state.mlStatusSnapshot; // We don't have it yet, trigger an update. if (!result) triggerStatusUpdate(); return result; @@ -387,15 +403,15 @@ const updateMLStatusSnapshot = async () => setMLStatusSnapshot(await getMLStatus()); const setMLStatusSnapshot = (snapshot: MLStatus) => { - _mlStatusSnapshot = snapshot; - _mlStatusListeners.forEach((l) => l()); + _state.mlStatusSnapshot = snapshot; + _state.mlStatusListeners.forEach((l) => l()); }; /** * Compute the current state of the ML subsystem. */ const getMLStatus = async (): Promise => { - if (!_isMLEnabled) return { phase: "disabled" }; + if (!_state.isMLEnabled) return { phase: "disabled" }; const { indexedCount, indexableCount } = await indexableAndIndexedCounts(); @@ -427,8 +443,11 @@ const getMLStatus = async (): Promise => { const setInterimScheduledStatus = () => { let nSyncedFiles = 0, nTotalFiles = 0; - if (_mlStatusSnapshot && _mlStatusSnapshot.phase != "disabled") { - ({ nSyncedFiles, nTotalFiles } = _mlStatusSnapshot); + if ( + _state.mlStatusSnapshot && + _state.mlStatusSnapshot.phase != "disabled" + ) { + ({ nSyncedFiles, nTotalFiles } = _state.mlStatusSnapshot); } setMLStatusSnapshot({ phase: "scheduled", nSyncedFiles, nTotalFiles }); }; From 69627ee8d6cc7efa0fd211c3d29a913372584714 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 11:14:23 +0530 Subject: [PATCH 101/211] Start moving the plumbing out --- web/packages/new/photos/services/ml/face.ts | 11 ++++++++ web/packages/new/photos/services/ml/index.ts | 28 ++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/web/packages/new/photos/services/ml/face.ts b/web/packages/new/photos/services/ml/face.ts index 7ecbf06002..891b605db2 100644 --- a/web/packages/new/photos/services/ml/face.ts +++ b/web/packages/new/photos/services/ml/face.ts @@ -139,6 +139,17 @@ export interface Face { * This ID is guaranteed to be unique for all the faces detected in all the * files for the user. In particular, each file can have multiple faces but * they all will get their own unique {@link faceID}. + * + * This ID is also meant to be stable across reindexing. That is, if the + * same algorithm and hyperparameters are used to reindex the file, then it + * should result in the same face IDs. This allows us leeway in letting + * unnecessary reindexing happen in rare cases without invalidating the + * clusters that rely on the presence of the given face ID. + * + * Finally, this face ID is not completely opaque. It consists of underscore + * separated components, the first of which is the ID of the + * {@link EnteFile} to which this face belongs. Client code can rely on this + * structure and can parse it if needed. */ faceID: string; /** diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 03e39343fb..584c447b40 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -3,6 +3,7 @@ */ import { isDesktop } from "@/base/app"; +import { assertionFailed } from "@/base/assert"; import { blobCache } from "@/base/blob-cache"; import { ensureElectron } from "@/base/electron"; import { isDevBuild } from "@/base/env"; @@ -69,6 +70,11 @@ class MLState { * See {@link mlStatusSnapshot}. */ mlStatusSnapshot: MLStatus | undefined; + + /** + * IDs files for which we are currently regenerating face crops. + */ + inFlightFaceCropRegenFileIDs = new Set(); } /** State shared by the functions in this module. See {@link MLState}. */ @@ -481,6 +487,28 @@ export const unidentifiedFaceIDs = async ( return index?.faces.map((f) => f.faceID) ?? []; }; +/** + * Return the cached face crop for the given face, regenerating it if needed. + * + * @param faceID The id of the face whose face crop we want. + */ +export const faceCrop = (faceID: string) => { + const fileID = fileIDFromFaceID(faceID); +}; + +/** + * Extract the ID of the {@link EnteFile} to which this face belongs from the + * given {@link faceID}. + */ +const fileIDFromFaceID = (faceID: string) => { + const fileID = parseInt(faceID.split("_")[0] ?? ""); + if (isNaN(fileID)) { + assertionFailed(`Ignoring attempt to parse invalid faceID ${faceID}`); + return undefined; + } + return fileID; +}; + /** * Check to see if any of the faces in the given file do not have a face crop * present locally. If so, then regenerate the face crops for all the faces in From a32a9dea3f27fdafa3bfa18e6042622a48452d55 Mon Sep 17 00:00:00 2001 From: Jay <118880285+Jishnuraj9@users.noreply.github.com> Date: Fri, 9 Aug 2024 11:18:17 +0530 Subject: [PATCH 102/211] [docs] updated googletakeout --- docs/docs/photos/migration/from-google-photos/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/docs/photos/migration/from-google-photos/index.md b/docs/docs/photos/migration/from-google-photos/index.md index 577a3283e5..feaaf7784a 100644 --- a/docs/docs/photos/migration/from-google-photos/index.md +++ b/docs/docs/photos/migration/from-google-photos/index.md @@ -60,3 +60,5 @@ If you run into any issues during this migration, please reach out to > this will not work is when Google has split the export into multiple parts, > and did not put the JSON file associated with an image in the same exported > zip. + +>So the best move is to unzip all of the items into a single root folder, and drop that folder into our desktop app. That way we have the complete picture, and can stitch together metadata with the correct files. \ No newline at end of file From 526546da594b2e63f0df066e62b6d4ef55341fbe Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 11:30:21 +0530 Subject: [PATCH 103/211] Promise --- web/packages/new/photos/services/ml/index.ts | 49 +++++++++----------- 1 file changed, 21 insertions(+), 28 deletions(-) diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 584c447b40..9c7191afd9 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -3,7 +3,6 @@ */ import { isDesktop } from "@/base/app"; -import { assertionFailed } from "@/base/assert"; import { blobCache } from "@/base/blob-cache"; import { ensureElectron } from "@/base/electron"; import { isDevBuild } from "@/base/env"; @@ -72,9 +71,10 @@ class MLState { mlStatusSnapshot: MLStatus | undefined; /** - * IDs files for which we are currently regenerating face crops. + * In flight face crop regeneration promises indexed by the IDs of the files + * whose faces we are regenerating. */ - inFlightFaceCropRegenFileIDs = new Set(); + inFlightFaceCropRegens = new Map>(); } /** State shared by the functions in this module. See {@link MLState}. */ @@ -491,43 +491,36 @@ export const unidentifiedFaceIDs = async ( * Return the cached face crop for the given face, regenerating it if needed. * * @param faceID The id of the face whose face crop we want. + * + * @param enteFile The {@link EnteFile} that contains this face. */ -export const faceCrop = (faceID: string) => { - const fileID = fileIDFromFaceID(faceID); -}; +export const faceCrop = async (faceID: string, enteFile: EnteFile) => { + let inFlight = _state.inFlightFaceCropRegens.get(enteFile.id); -/** - * Extract the ID of the {@link EnteFile} to which this face belongs from the - * given {@link faceID}. - */ -const fileIDFromFaceID = (faceID: string) => { - const fileID = parseInt(faceID.split("_")[0] ?? ""); - if (isNaN(fileID)) { - assertionFailed(`Ignoring attempt to parse invalid faceID ${faceID}`); - return undefined; + if (!inFlight) { + inFlight = regenerateFaceCropsIfNeeded(enteFile); + _state.inFlightFaceCropRegens.set(enteFile.id, inFlight); } - return fileID; + + await inFlight; + + const cache = await blobCache("face-crops"); + return cache.get(faceID); }; /** * Check to see if any of the faces in the given file do not have a face crop * present locally. If so, then regenerate the face crops for all the faces in * the file (updating the "face-crops" {@link BlobCache}). - * - * @returns true if one or more face crops were regenerated; false otherwise. */ -export const regenerateFaceCropsIfNeeded = async (enteFile: EnteFile) => { +const regenerateFaceCropsIfNeeded = async (enteFile: EnteFile) => { const index = await faceIndex(enteFile.id); - if (!index) return false; + if (!index) return; - const faceIDs = index.faces.map((f) => f.faceID); const cache = await blobCache("face-crops"); - for (const id of faceIDs) { - if (!(await cache.has(id))) { - await regenerateFaceCrops(enteFile, index); - return true; - } - } + const faceIDs = index.faces.map((f) => f.faceID); + let needsRegen = false; + for (const id of faceIDs) if (!(await cache.has(id))) needsRegen = true; - return false; + if (needsRegen) await regenerateFaceCrops(enteFile, index); }; From 771327a551ae4cb5efb9d202cfe19bc15b239b64 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 11:35:50 +0530 Subject: [PATCH 104/211] gen --- .../new/photos/components/PeopleList.tsx | 49 ++++++++----------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/web/packages/new/photos/components/PeopleList.tsx b/web/packages/new/photos/components/PeopleList.tsx index 960687d7c2..30780cbc44 100644 --- a/web/packages/new/photos/components/PeopleList.tsx +++ b/web/packages/new/photos/components/PeopleList.tsx @@ -1,8 +1,4 @@ -import { blobCache } from "@/base/blob-cache"; -import { - regenerateFaceCropsIfNeeded, - unidentifiedFaceIDs, -} from "@/new/photos/services/ml"; +import { faceCrop, unidentifiedFaceIDs } from "@/new/photos/services/ml"; import type { Person } from "@/new/photos/services/ml/people"; import type { EnteFile } from "@/new/photos/types/file"; import { Skeleton, Typography, styled } from "@mui/material"; @@ -80,7 +76,6 @@ export const UnidentifiedFaces: React.FC = ({ enteFile, }) => { const [faceIDs, setFaceIDs] = useState([]); - const [didRegen, setDidRegen] = useState(false); useEffect(() => { let didCancel = false; @@ -88,13 +83,6 @@ export const UnidentifiedFaces: React.FC = ({ const go = async () => { const faceIDs = await unidentifiedFaceIDs(enteFile); !didCancel && setFaceIDs(faceIDs); - // Don't block for the regeneration to happen. If anything got - // regenerated, the result will be true, in response to which we'll - // change the key of the face list and cause it to be rerendered - // (fetching the regenerated crops). - void regenerateFaceCropsIfNeeded(enteFile).then((r) => - setDidRegen(r), - ); }; void go(); @@ -111,10 +99,10 @@ export const UnidentifiedFaces: React.FC = ({ {t("UNIDENTIFIED_FACES")} - + {faceIDs.map((faceID) => ( - + ))} @@ -123,30 +111,33 @@ export const UnidentifiedFaces: React.FC = ({ }; interface FaceCropImageViewProps { + /** The {@link EnteFile} which contains this face. */ + enteFile: EnteFile; + /** The ID of the face to display. */ faceID: string; } /** - * An image view showing the face crop for the given {@link faceID}. + * An image view showing the face crop for the given face. + * + * The image is read from the "face-crops" {@link BlobCache}, regenerating it if + * needed (which is why also need to pass the associated file). * - * The image is read from the "face-crops" {@link BlobCache}. While the image is - * being fetched, or if it doesn't exist, a placeholder is shown. + * While the image is being fetched, or if it doesn't exist, a placeholder is + * shown. */ -const FaceCropImageView: React.FC = ({ faceID }) => { +const FaceCropImageView: React.FC = ({ + enteFile, + faceID, +}) => { const [objectURL, setObjectURL] = useState(); useEffect(() => { let didCancel = false; - if (faceID) { - void blobCache("face-crops") - .then((cache) => cache.get(faceID)) - .then((data) => { - if (data) { - const blob = new Blob([data]); - if (!didCancel) setObjectURL(URL.createObjectURL(blob)); - } - }); - } else setObjectURL(undefined); + + void faceCrop(faceID, enteFile).then((blob) => { + if (blob && !didCancel) setObjectURL(URL.createObjectURL(blob)); + }); return () => { didCancel = true; From 5c7c4ad35acf078788acbdd29ea1958381165b99 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 11:38:52 +0530 Subject: [PATCH 105/211] Fix --- web/packages/base/assert.ts | 11 ++++++++++ .../new/photos/components/PeopleList.tsx | 21 +++++++++---------- web/packages/new/photos/services/ml/index.ts | 13 ++++++++++++ 3 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 web/packages/base/assert.ts diff --git a/web/packages/base/assert.ts b/web/packages/base/assert.ts new file mode 100644 index 0000000000..e20b7d1e43 --- /dev/null +++ b/web/packages/base/assert.ts @@ -0,0 +1,11 @@ +import { isDevBuild } from "./env"; +import log from "./log"; + +/** + * If running in a dev build, throw an exception with the given message. + * Otherwise log it as a warning. + */ +export const assertionFailed = (message: string) => { + if (isDevBuild) throw new Error(message); + log.warn(message); +}; diff --git a/web/packages/new/photos/components/PeopleList.tsx b/web/packages/new/photos/components/PeopleList.tsx index 30780cbc44..35475ae689 100644 --- a/web/packages/new/photos/components/PeopleList.tsx +++ b/web/packages/new/photos/components/PeopleList.tsx @@ -111,10 +111,10 @@ export const UnidentifiedFaces: React.FC = ({ }; interface FaceCropImageViewProps { - /** The {@link EnteFile} which contains this face. */ - enteFile: EnteFile; /** The ID of the face to display. */ faceID: string; + /** The {@link EnteFile} which contains this face. */ + enteFile: EnteFile; } /** @@ -123,30 +123,29 @@ interface FaceCropImageViewProps { * The image is read from the "face-crops" {@link BlobCache}, regenerating it if * needed (which is why also need to pass the associated file). * - * While the image is being fetched, or if it doesn't exist, a placeholder is - * shown. + * While the image is being fetched or regenerated, or if it doesn't exist, a + * placeholder is shown. */ const FaceCropImageView: React.FC = ({ - enteFile, faceID, + enteFile, }) => { const [objectURL, setObjectURL] = useState(); useEffect(() => { let didCancel = false; + let thisObjectURL: string | undefined; void faceCrop(faceID, enteFile).then((blob) => { - if (blob && !didCancel) setObjectURL(URL.createObjectURL(blob)); + if (blob && !didCancel) + setObjectURL((thisObjectURL = URL.createObjectURL(blob))); }); return () => { didCancel = true; - if (objectURL) URL.revokeObjectURL(objectURL); + if (thisObjectURL) URL.revokeObjectURL(thisObjectURL); }; - // TODO: The linter warning is actually correct, objectURL should be a - // dependency, but adding that require reworking this code first. - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [faceID]); + }, [faceID, enteFile]); return objectURL ? ( diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 9c7191afd9..ae9db79989 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -4,6 +4,7 @@ import { isDesktop } from "@/base/app"; import { blobCache } from "@/base/blob-cache"; +import { assertionFailed } from "@/base/assert"; import { ensureElectron } from "@/base/electron"; import { isDevBuild } from "@/base/env"; import log from "@/base/log"; @@ -487,6 +488,18 @@ export const unidentifiedFaceIDs = async ( return index?.faces.map((f) => f.faceID) ?? []; }; +/** + * Extract the ID of the {@link EnteFile} to which a face belongs from its ID. + */ +export const fileIDFromFaceID = (faceID: string) => { + const fileID = parseInt(faceID.split("_")[0] ?? ""); + if (isNaN(fileID)) { + assertionFailed(`Ignoring attempt to parse invalid faceID ${faceID}`); + return undefined; + } + return fileID; +}; + /** * Return the cached face crop for the given face, regenerating it if needed. * From da09a5261bd0a3858f444a9382a27222e8a3c845 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 9 Aug 2024 11:57:21 +0530 Subject: [PATCH 106/211] [server] Fix key look up --- server/configurations/local.yaml | 2 +- server/pkg/utils/s3config/filedata.go | 10 ++++++---- server/pkg/utils/s3config/s3config.go | 11 ++++++++++- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/server/configurations/local.yaml b/server/configurations/local.yaml index 9e26d820f8..1adad9e775 100644 --- a/server/configurations/local.yaml +++ b/server/configurations/local.yaml @@ -174,7 +174,7 @@ s3: #use_path_style_urls: true # # file-data-storage: - # derivedMetadata: + # derivedMeta: # primaryBucket: # replicaBuckets: [] # img_preview: diff --git a/server/pkg/utils/s3config/filedata.go b/server/pkg/utils/s3config/filedata.go index a39501ba40..fca1ecbcf3 100644 --- a/server/pkg/utils/s3config/filedata.go +++ b/server/pkg/utils/s3config/filedata.go @@ -3,6 +3,7 @@ package s3config import ( "fmt" "github.com/ente-io/museum/ente" + "strings" ) type ObjectBucketConfig struct { @@ -11,19 +12,20 @@ type ObjectBucketConfig struct { } type FileDataConfig struct { - ObjectBucketConfig map[ente.ObjectType]ObjectBucketConfig `mapstructure:"file-data-config"` + ObjectBucketConfig map[string]ObjectBucketConfig `mapstructure:"file-data-config"` } func (f FileDataConfig) HasConfig(objectType ente.ObjectType) bool { if objectType == "" || objectType == ente.FILE || objectType == ente.THUMBNAIL { panic(fmt.Sprintf("Invalid object type: %s", objectType)) } - _, ok := f.ObjectBucketConfig[objectType] + + _, ok := f.ObjectBucketConfig[strings.ToLower(string(objectType))] return ok } func (f FileDataConfig) GetPrimaryBucketID(objectType ente.ObjectType) string { - config, ok := f.ObjectBucketConfig[objectType] + config, ok := f.ObjectBucketConfig[strings.ToLower(string(objectType))] if !ok { panic(fmt.Sprintf("No config for object type: %s, use HasConfig", objectType)) } @@ -31,7 +33,7 @@ func (f FileDataConfig) GetPrimaryBucketID(objectType ente.ObjectType) string { } func (f FileDataConfig) GetReplicaBuckets(objectType ente.ObjectType) []string { - config, ok := f.ObjectBucketConfig[objectType] + config, ok := f.ObjectBucketConfig[strings.ToLower(string(objectType))] if !ok { panic(fmt.Sprintf("No config for object type: %s, use HasConfig", objectType)) } diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index b6ec9ba2af..b027ffac35 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -174,7 +174,16 @@ func (config *S3Config) GetBucketID(oType ente.ObjectType) string { if oType == ente.DerivedMeta || oType == ente.PreviewVideo || oType == ente.PreviewImage { return config.derivedStorageDC } - panic(fmt.Sprintf("No bucket for object type: %s", oType)) + panic(fmt.Sprintf("ops not supported for type: %s", oType)) +} +func (config *S3Config) GetReplicatedBuckets(oType ente.ObjectType) []string { + if config.fileDataConfig.HasConfig(oType) { + return config.fileDataConfig.GetReplicaBuckets(oType) + } + if oType == ente.DerivedMeta || oType == ente.PreviewVideo || oType == ente.PreviewImage { + return []string{} + } + panic(fmt.Sprintf("ops not supported for object type: %s", oType)) } func (config *S3Config) IsBucketActive(bucketID string) bool { From cfcfade152407b00f4095f8c4eeeb13539d24b6b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 12:00:31 +0530 Subject: [PATCH 107/211] People list --- .../new/photos/components/PeopleList.tsx | 5 ++++- web/packages/new/photos/services/ml/index.ts | 19 +++++++++++++++---- web/packages/new/photos/services/ml/people.ts | 5 ++++- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/web/packages/new/photos/components/PeopleList.tsx b/web/packages/new/photos/components/PeopleList.tsx index 35475ae689..adc53dbaf2 100644 --- a/web/packages/new/photos/components/PeopleList.tsx +++ b/web/packages/new/photos/components/PeopleList.tsx @@ -24,7 +24,10 @@ export const PeopleList: React.FC = ({ clickable={!!onSelect} onClick={() => onSelect && onSelect(person, index)} > - + ))} diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index ae9db79989..434568e22f 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -3,8 +3,8 @@ */ import { isDesktop } from "@/base/app"; -import { blobCache } from "@/base/blob-cache"; import { assertionFailed } from "@/base/assert"; +import { blobCache } from "@/base/blob-cache"; import { ensureElectron } from "@/base/electron"; import { isDevBuild } from "@/base/env"; import log from "@/base/log"; @@ -16,6 +16,7 @@ import { ensure } from "@/utils/ensure"; import { throttled } from "@/utils/promise"; import { proxy, transfer } from "comlink"; import { isInternalUser } from "../feature-flags"; +import { getAllLocalFiles } from "../files"; import { getRemoteFlag, updateRemoteFlag } from "../remote-store"; import type { UploadItem } from "../upload/types"; import { clusterFaces } from "./cluster-new"; @@ -321,15 +322,25 @@ export const wipCluster = async () => { const clusters = clusterFaces(await faceIndexes()); + const localFiles = await getAllLocalFiles(); + const localFilesByID = new Map(localFiles.map((f) => [f.id, f])); + const people: Person[] = []; // await mlIDbStorage.getAllPeople(); for (const cluster of clusters) { + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const dfID = cluster.faceIDs[0]!; + const dfFile = localFilesByID.get(fileIDFromFaceID(dfID) ?? 0); + if (!dfFile) { + assertionFailed(`Face ID ${dfID} without local file`); + continue; + } people.push({ id: Math.random(), //cluster.id, name: "test", // eslint-disable-next-line @typescript-eslint/no-non-null-assertion files: cluster.faceIDs.map((s) => parseInt(s.split("_")[0]!)), - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - displayFaceId: cluster.faceIDs[0]!, + displayFaceID: dfID, + displayFaceFile: dfFile, }); } @@ -491,7 +502,7 @@ export const unidentifiedFaceIDs = async ( /** * Extract the ID of the {@link EnteFile} to which a face belongs from its ID. */ -export const fileIDFromFaceID = (faceID: string) => { +const fileIDFromFaceID = (faceID: string) => { const fileID = parseInt(faceID.split("_")[0] ?? ""); if (isNaN(fileID)) { assertionFailed(`Ignoring attempt to parse invalid faceID ${faceID}`); diff --git a/web/packages/new/photos/services/ml/people.ts b/web/packages/new/photos/services/ml/people.ts index 90e1314571..a85cc2f20a 100644 --- a/web/packages/new/photos/services/ml/people.ts +++ b/web/packages/new/photos/services/ml/people.ts @@ -1,8 +1,11 @@ +import type { EnteFile } from "../../types/file"; + export interface Person { id: number; name?: string; files: number[]; - displayFaceId: string; + displayFaceID: string; + displayFaceFile: EnteFile; } // Forced disable clustering. It doesn't currently work. From 890ea6c8d16f683140685b1c6300710147b7be32 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 12:22:55 +0530 Subject: [PATCH 108/211] Closer --- web/apps/photos/src/services/searchService.ts | 4 ++- .../new/photos/services/ml/cluster-new.ts | 32 +++++++++++-------- web/packages/new/photos/services/ml/index.ts | 4 +-- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 8488462aba..c697c34575 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -421,7 +421,9 @@ async function getAllPeople(limit: number = undefined) { // people.push(person); // }); people = people ?? []; - return people + const result = people .sort((p1, p2) => p2.files.length - p1.files.length) .slice(0, limit); + // log.debug(() => ["getAllPeople", result]); + return result; } diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index bee7c2fee8..3a615471d4 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -81,34 +81,40 @@ export const clusterFaces = (faceIndexes: FaceIndex[]) => { const clusters: FaceCluster[] = []; const clusterIndexByFaceID = new Map(); for (const [i, { faceID, embedding }] of faces.entries()) { - let j = 0; - for (; j < i; j++) { - // Can't find a better way for avoiding the null assertion. + // Find the nearest neighbour from among the faces we have already seen. + let nnIndex: number | undefined; + let nnCosineSimilarity = 0; + for (let j = 0; j < i; j++) { + // Can't find a way of avoiding the null assertion. // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const n = faces[j]!; - // TODO-ML: The distance metric and the thresholds are placeholders. - // The vectors are already normalized, so we can directly use their // dot product as their cosine similarity. const csim = dotProduct(embedding, n.embedding); - if (csim > 0.5) { - // Found a neighbour near enough. Add this face to the - // neighbour's cluster and call it a day. - const ci = ensure(clusterIndexByFaceID.get(n.faceID)); - clusters[ci]?.faceIDs.push(faceID); - clusterIndexByFaceID.set(faceID, ci); - break; + if (csim > 0.76 && csim > nnCosineSimilarity) { + nnIndex = j; + nnCosineSimilarity = csim; } } - if (j == i) { + if (nnIndex === undefined) { // We didn't find a neighbour. Create a new cluster with this face. + const cluster = { id: newNonSecureID("cluster_"), faceIDs: [faceID], }; clusters.push(cluster); clusterIndexByFaceID.set(faceID, clusters.length); + } else { + // Found a neighbour near enough. Add this face to the neighbour's + // cluster. + + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const nn = faces[nnIndex]!; + const nnClusterIndex = ensure(clusterIndexByFaceID.get(nn.faceID)); + clusters[nnClusterIndex]?.faceIDs.push(faceID); + clusterIndexByFaceID.set(faceID, nnClusterIndex); } } diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 434568e22f..bee9291563 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -309,7 +309,7 @@ export const indexNewUpload = (enteFile: EnteFile, uploadItem: UploadItem) => { void worker().then((w) => w.onUpload(enteFile, uploadItem)); }; -let last: Person[] = []; +let last: Person[] | undefined; /** * WIP! Don't enable, dragon eggs are hatching here. @@ -318,7 +318,7 @@ export const wipCluster = async () => { if (!isDevBuild || !(await isInternalUser())) return; if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return; - if (last.length) return last; + if (last) return last; const clusters = clusterFaces(await faceIndexes()); From 762daa6bd51e4d7b3d56a889b5676ad285df8ed2 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 9 Aug 2024 12:33:45 +0530 Subject: [PATCH 109/211] [mob][photos] Change icon of guest view --- mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart | 2 +- mobile/lib/ui/viewer/file/file_app_bar.dart | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index 148d97f5e6..1ab3d0fb5b 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -274,7 +274,7 @@ class _FileSelectionActionsWidgetState } items.add( SelectionActionButton( - icon: Icons.lock, + icon: Icons.people_outline_rounded, labelText: "Guest view", onTap: _onGuestViewClick, ), diff --git a/mobile/lib/ui/viewer/file/file_app_bar.dart b/mobile/lib/ui/viewer/file/file_app_bar.dart index 28ba653e2c..6120ffb5df 100644 --- a/mobile/lib/ui/viewer/file/file_app_bar.dart +++ b/mobile/lib/ui/viewer/file/file_app_bar.dart @@ -299,7 +299,7 @@ class FileAppBarState extends State { child: Row( children: [ Icon( - Icons.lock, + Icons.people_outline_rounded, color: Theme.of(context).iconTheme.color, ), const Padding( From ffbd76b88b9b2d95dbe87f7650cadf365146961e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 9 Aug 2024 12:40:52 +0530 Subject: [PATCH 110/211] [server] Support for replicating ml data --- server/cmd/museum/main.go | 5 + server/migrations/89_file_data_table.up.sql | 2 +- server/pkg/controller/filedata/controller.go | 11 +- server/pkg/controller/filedata/replicate.go | 115 ++++++++++++++++++- server/pkg/controller/replication3.go | 21 +--- server/pkg/repo/filedata/repository.go | 32 ++++++ server/pkg/utils/file/file.go | 18 +++ 7 files changed, 178 insertions(+), 26 deletions(-) diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index 8c9c88f304..f74b004421 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -831,6 +831,11 @@ func setupAndStartBackgroundJobs( } else { log.Info("Skipping Replication as replication is disabled") } + + err := fileDataCtrl.StartReplication() + if err != nil { + log.Warnf("Could not start fileData replication: %s", err) + } fileDataCtrl.StartDataDeletion() // Start data deletion for file data; objectCleanupController.StartRemovingUnreportedObjects() diff --git a/server/migrations/89_file_data_table.up.sql b/server/migrations/89_file_data_table.up.sql index c0feab6e64..72bfe0979d 100644 --- a/server/migrations/89_file_data_table.up.sql +++ b/server/migrations/89_file_data_table.up.sql @@ -15,7 +15,7 @@ CREATE TABLE IF NOT EXISTS file_data delete_from_buckets s3region[] NOT NULL DEFAULT '{}', inflight_rep_buckets s3region[] NOT NULL DEFAULT '{}', is_deleted BOOLEAN NOT NULL DEFAULT false, - pending_sync BOOLEAN NOT NULL DEFAULT false, + pending_sync BOOLEAN NOT NULL DEFAULT true, sync_locked_till BIGINT NOT NULL DEFAULT 0, created_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), updated_at BIGINT NOT NULL DEFAULT now_utc_micro_seconds(), diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index fb795c01c0..afdd895854 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -55,6 +55,9 @@ type Controller struct { HostName string cleanupCronRunning bool downloadManagerCache map[string]*s3manager.Downloader + + // for downloading objects from s3 for replication + workerURL string } func New(repo *fileDataRepo.Repository, @@ -185,8 +188,8 @@ func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row) ([]bulkS3M dc := row.LatestBucket s3FileMetadata, err := c.fetchS3FileMetadata(context.Background(), row, dc) if err != nil { - log.WithField("bucket", row.LatestBucket). - Error("error fetching embedding object: "+row.S3FileMetadataObjectKey(), err) + log.WithField("bucket", dc). + Error("error fetching object: "+row.S3FileMetadataObjectKey(), err) embeddingObjects[i] = bulkS3MetaFetchResult{ err: err, dbEntry: row, @@ -204,7 +207,7 @@ func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row) ([]bulkS3M return embeddingObjects, nil } -func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, _ string) (*fileData.S3FileMetadata, error) { +func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, dc string) (*fileData.S3FileMetadata, error) { opt := _defaultFetchConfig objectKey := row.S3FileMetadataObjectKey() ctxLogger := log.WithField("objectKey", objectKey).WithField("dc", row.LatestBucket) @@ -223,7 +226,7 @@ func (c *Controller) fetchS3FileMetadata(ctx context.Context, row fileData.Row, cancel() return nil, stacktrace.Propagate(ctx.Err(), "") default: - obj, err := c.downloadObject(fetchCtx, objectKey, row.LatestBucket) + obj, err := c.downloadObject(fetchCtx, objectKey, dc) cancel() // Ensure cancel is called to release resources if err == nil { if i > 0 { diff --git a/server/pkg/controller/filedata/replicate.go b/server/pkg/controller/filedata/replicate.go index 7baa814199..1cf4488d70 100644 --- a/server/pkg/controller/filedata/replicate.go +++ b/server/pkg/controller/filedata/replicate.go @@ -1,6 +1,119 @@ package filedata +import ( + "context" + "database/sql" + "errors" + "fmt" + "github.com/ente-io/museum/ente/filedata" + fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata" + enteTime "github.com/ente-io/museum/pkg/utils/time" + "github.com/ente-io/stacktrace" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "time" +) + +// StartReplication starts the replication process for file data. +// If func (c *Controller) StartReplication() error { - // todo: implement replication logic + workerURL := viper.GetString("replication.worker-url") + if workerURL == "" { + log.Infof("replication.worker-url was not defined, file data will downloaded directly during replication") + } else { + log.Infof("Worker URL to download objects for replication v3 is: %s", workerURL) + } + c.workerURL = workerURL + + workerCount := viper.GetInt("replication.file-data.worker-count") + if workerCount == 0 { + workerCount = 6 + } + go c.startWorkers(workerCount) return nil } +func (c *Controller) startWorkers(n int) { + log.Infof("Starting %d workers for replication v3", n) + + for i := 0; i < n; i++ { + go c.replicate(i) + // Stagger the workers + time.Sleep(time.Duration(2*i+1) * time.Second) + } +} + +// Entry point for the replication worker (goroutine) +// +// i is an arbitrary index of the current routine. +func (c *Controller) replicate(i int) { + for { + err := c.tryReplicate() + if err != nil { + // Sleep in proportion to the (arbitrary) index to space out the + // workers further. + time.Sleep(time.Duration(i+1) * time.Minute) + } + } +} + +func (c *Controller) tryReplicate() error { + newLockTime := enteTime.MicrosecondsAfterMinutes(60) + ctx, cancelFun := context.WithTimeout(context.Background(), 50*time.Minute) + defer cancelFun() + row, err := c.Repo.GetPendingSyncDataAndExtendLock(ctx, newLockTime, false) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + log.Errorf("Could not fetch row for deletion: %s", err) + } + return err + } + err = c.replicateRowData(ctx, *row) + if err != nil { + log.Errorf("Could not delete file data: %s", err) + return err + } else { + // If the replication was completed without any errors, we can reset the lock time + return c.Repo.ResetSyncLock(ctx, *row, newLockTime) + } +} + +func (c *Controller) replicateRowData(ctx context.Context, row filedata.Row) error { + wantInBucketIDs := map[string]bool{} + wantInBucketIDs[c.S3Config.GetBucketID(row.Type)] = true + rep := c.S3Config.GetReplicatedBuckets(row.Type) + for _, bucket := range rep { + wantInBucketIDs[bucket] = true + } + delete(wantInBucketIDs, row.LatestBucket) + for _, bucket := range row.ReplicatedBuckets { + delete(wantInBucketIDs, bucket) + } + if len(wantInBucketIDs) > 0 { + s3FileMetadata, err := c.downloadObject(ctx, row.S3FileMetadataObjectKey(), row.LatestBucket) + if err != nil { + return stacktrace.Propagate(err, "error fetching metadata object "+row.S3FileMetadataObjectKey()) + } + for bucketID := range wantInBucketIDs { + if err := c.uploadAndVerify(ctx, row, s3FileMetadata, bucketID); err != nil { + return stacktrace.Propagate(err, "error uploading and verifying metadata object") + } + } + } else { + log.Infof("No replication pending for file %d and type %s", row.FileID, string(row.Type)) + } + return c.Repo.MarkReplicationAsDone(ctx, row) +} + +func (c *Controller) uploadAndVerify(ctx context.Context, row filedata.Row, s3FileMetadata filedata.S3FileMetadata, dstBucketID string) error { + if err := c.Repo.RegisterReplicationAttempt(ctx, row, dstBucketID); err != nil { + return stacktrace.Propagate(err, "could not register replication attempt") + } + metadataSize, err := c.uploadObject(s3FileMetadata, row.S3FileMetadataObjectKey(), dstBucketID) + if err != nil { + return err + } + if metadataSize != row.Size { + return fmt.Errorf("uploaded metadata size %d does not match expected size %d", metadataSize, row.Size) + } + return c.Repo.MoveBetweenBuckets(row, dstBucketID, fileDataRepo.InflightRepColumn, fileDataRepo.ReplicationColumn) +} diff --git a/server/pkg/controller/replication3.go b/server/pkg/controller/replication3.go index 4fad173ea2..6fe1c0f2bc 100644 --- a/server/pkg/controller/replication3.go +++ b/server/pkg/controller/replication3.go @@ -308,7 +308,7 @@ func (c *ReplicationController3) tryReplicate() error { return commit(nil) } - err = ensureSufficientSpace(ob.Size) + err = file.EnsureSufficientSpace(ob.Size) if err != nil { // We don't have free space right now, maybe because other big files are // being downloaded simultanously, but we might get space later, so mark @@ -360,25 +360,6 @@ func (c *ReplicationController3) tryReplicate() error { return commit(err) } -// Return an error if we risk running out of disk space if we try to download -// and write a file of size. -// -// This function keeps a buffer of 1 GB free space in its calculations. -func ensureSufficientSpace(size int64) error { - free, err := file.FreeSpace("/") - if err != nil { - return stacktrace.Propagate(err, "Failed to fetch free space") - } - - gb := uint64(1024) * 1024 * 1024 - need := uint64(size) + (2 * gb) - if free < need { - return fmt.Errorf("insufficient space on disk (need %d bytes, free %d bytes)", size, free) - } - - return nil -} - // Create a temporary file for storing objectKey. Return both the path to the // file, and the handle to the file. // diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go index bdf184591e..0293b05f0f 100644 --- a/server/pkg/repo/filedata/repository.go +++ b/server/pkg/repo/filedata/repository.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/ente-io/museum/ente" "github.com/ente-io/museum/ente/filedata" + "github.com/ente-io/museum/pkg/utils/array" "github.com/ente-io/stacktrace" "github.com/lib/pq" "time" @@ -194,6 +195,37 @@ func (r *Repository) GetPendingSyncDataAndExtendLock(ctx context.Context, newSyn return &fileData, nil } +// MarkReplicationAsDone marks the pending_sync as false for the file data row +func (r *Repository) MarkReplicationAsDone(ctx context.Context, row filedata.Row) error { + query := `UPDATE file_data SET pending_sync = false WHERE is_deleted=true and file_id = $1 AND data_type = $2 AND user_id = $3` + _, err := r.DB.ExecContext(ctx, query, row.FileID, string(row.Type), row.UserID) + if err != nil { + return stacktrace.Propagate(err, "") + } + return nil +} + +func (r *Repository) RegisterReplicationAttempt(ctx context.Context, row filedata.Row, dstBucketID string) error { + if array.StringInList(dstBucketID, row.DeleteFromBuckets) { + return r.MoveBetweenBuckets(row, dstBucketID, DeletionColumn, InflightRepColumn) + } + if array.StringInList(dstBucketID, row.InflightReplicas) == false { + return r.AddBucket(row, dstBucketID, InflightRepColumn) + } + return nil +} + +// ResetSyncLock resets the sync_locked_till to now_utc_micro_seconds() for the file data row only if pending_sync is false and +// the input syncLockedTill is equal to the existing sync_locked_till. This is used to reset the lock after the replication is done +func (r *Repository) ResetSyncLock(ctx context.Context, row filedata.Row, syncLockedTill int64) error { + query := `UPDATE file_data SET sync_locked_till = now_utc_micro_seconds() WHERE pending_sync = false and file_id = $1 AND data_type = $2 AND user_id = $3 AND sync_locked_till = $4` + _, err := r.DB.ExecContext(ctx, query, row.FileID, string(row.Type), row.UserID, syncLockedTill) + if err != nil { + return stacktrace.Propagate(err, "") + } + return nil +} + func (r *Repository) DeleteFileData(ctx context.Context, row filedata.Row) error { query := ` DELETE FROM file_data diff --git a/server/pkg/utils/file/file.go b/server/pkg/utils/file/file.go index db94347026..5e1872e59c 100644 --- a/server/pkg/utils/file/file.go +++ b/server/pkg/utils/file/file.go @@ -35,6 +35,24 @@ func FreeSpace(path string) (uint64, error) { return fs.Bfree * uint64(fs.Bsize), nil } +// EnsureSufficientSpace Return an error if we risk running out of disk space if we try to download +// and write a file of size. +// This function keeps a buffer of 2 GB free space in its calculations. +func EnsureSufficientSpace(size int64) error { + free, err := FreeSpace("/") + if err != nil { + return stacktrace.Propagate(err, "Failed to fetch free space") + } + + gb := uint64(1024) * 1024 * 1024 + need := uint64(size) + (2 * gb) + if free < need { + return fmt.Errorf("insufficient space on disk (need %d bytes, free %d bytes)", size, free) + } + + return nil +} + func GetLockNameForObject(objectKey string) string { return fmt.Sprintf("Object:%s", objectKey) } From 46a7880f050a4d66ab3886b13d1304d9beee5274 Mon Sep 17 00:00:00 2001 From: ashilkn Date: Fri, 9 Aug 2024 12:41:04 +0530 Subject: [PATCH 111/211] [mob][photos] Extract strings --- mobile/lib/generated/intl/messages_cs.dart | 1 + mobile/lib/generated/intl/messages_de.dart | 37 ++++++++++--------- mobile/lib/generated/intl/messages_en.dart | 1 + mobile/lib/generated/intl/messages_es.dart | 1 + mobile/lib/generated/intl/messages_fr.dart | 1 + mobile/lib/generated/intl/messages_it.dart | 1 + mobile/lib/generated/intl/messages_ko.dart | 1 + mobile/lib/generated/intl/messages_nl.dart | 1 + mobile/lib/generated/intl/messages_no.dart | 1 + mobile/lib/generated/intl/messages_pl.dart | 11 +++--- mobile/lib/generated/intl/messages_pt.dart | 11 +++--- mobile/lib/generated/intl/messages_ru.dart | 1 + mobile/lib/generated/intl/messages_tr.dart | 1 + mobile/lib/generated/intl/messages_zh.dart | 16 ++++---- mobile/lib/generated/l10n.dart | 10 +++++ mobile/lib/l10n/intl_cs.arb | 3 +- mobile/lib/l10n/intl_de.arb | 3 +- mobile/lib/l10n/intl_en.arb | 3 +- mobile/lib/l10n/intl_es.arb | 3 +- mobile/lib/l10n/intl_fr.arb | 3 +- mobile/lib/l10n/intl_it.arb | 3 +- mobile/lib/l10n/intl_ko.arb | 3 +- mobile/lib/l10n/intl_nl.arb | 3 +- mobile/lib/l10n/intl_no.arb | 3 +- mobile/lib/l10n/intl_pl.arb | 3 +- mobile/lib/l10n/intl_pt.arb | 3 +- mobile/lib/l10n/intl_ru.arb | 3 +- mobile/lib/l10n/intl_tr.arb | 3 +- mobile/lib/l10n/intl_zh.arb | 3 +- .../file_selection_actions_widget.dart | 2 +- mobile/lib/ui/viewer/file/file_app_bar.dart | 2 +- 31 files changed, 88 insertions(+), 53 deletions(-) diff --git a/mobile/lib/generated/intl/messages_cs.dart b/mobile/lib/generated/intl/messages_cs.dart index 0b75d517e1..6c8d749cc6 100644 --- a/mobile/lib/generated/intl/messages_cs.dart +++ b/mobile/lib/generated/intl/messages_cs.dart @@ -60,6 +60,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Face recognition"), "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hideContent": MessageLookupByLibrary.simpleMessage("Hide content"), "hideContentDescriptionAndroid": MessageLookupByLibrary.simpleMessage( "Hides app content in the app switcher and disables screenshots"), diff --git a/mobile/lib/generated/intl/messages_de.dart b/mobile/lib/generated/intl/messages_de.dart index 5f340299ae..6987ee0830 100644 --- a/mobile/lib/generated/intl/messages_de.dart +++ b/mobile/lib/generated/intl/messages_de.dart @@ -136,7 +136,7 @@ class MessageLookup extends MessageLookupByLibrary { "Bitte kontaktiere den Support von ${providerName}, falls etwas abgebucht wurde"; static String m40(endDate) => - "Kostenlose Testversion gültig bis ${endDate}.\nSie können anschließend ein bezahltes Paket auswählen."; + "Kostenlose Testversion gültig bis ${endDate}.\nDu kannst anschließend ein bezahltes Paket auswählen."; static String m41(toEmail) => "Bitte sende uns eine E-Mail an ${toEmail}"; @@ -191,7 +191,7 @@ class MessageLookup extends MessageLookupByLibrary { static String m60(id) => "Dein ${id} ist bereits mit einem anderen Ente-Konto verknüpft.\nWenn du deine ${id} mit diesem Konto verwenden möchtest, kontaktiere bitte unseren Support"; - static String m61(endDate) => "Ihr Abo endet am ${endDate}"; + static String m61(endDate) => "Dein Abo endet am ${endDate}"; static String m62(completed, total) => "${completed}/${total} Erinnerungsstücke gesichert"; @@ -343,12 +343,12 @@ class MessageLookup extends MessageLookupByLibrary { "askDeleteReason": MessageLookupByLibrary.simpleMessage( "Was ist der Hauptgrund für die Löschung deines Kontos?"), "askYourLovedOnesToShare": MessageLookupByLibrary.simpleMessage( - "Bitte deine Liebsten ums teilen"), + "Bitte deine Liebsten ums Teilen"), "atAFalloutShelter": MessageLookupByLibrary.simpleMessage( "in einem ehemaligen Luftschutzbunker"), "authToChangeEmailVerificationSetting": MessageLookupByLibrary.simpleMessage( - "Bitte Authentifizieren um die E-Mail Bestätigung zu ändern"), + "Bitte authentifizieren, um die E-Mail-Bestätigung zu ändern"), "authToChangeLockscreenSetting": MessageLookupByLibrary.simpleMessage( "Bitte authentifizieren, um die Sperrbildschirm-Einstellung zu ändern"), "authToChangeYourEmail": MessageLookupByLibrary.simpleMessage( @@ -383,7 +383,7 @@ class MessageLookup extends MessageLookupByLibrary { "autoLockFeatureDescription": MessageLookupByLibrary.simpleMessage( "Zeit, nach der die App gesperrt wird, nachdem sie in den Hintergrund verschoben wurde"), "autoLogoutMessage": MessageLookupByLibrary.simpleMessage( - "Aufgrund technischer Störungen wurden Sie abgemeldet. Wir entschuldigen uns für die Unannehmlichkeiten."), + "Du wurdest aufgrund technischer Störungen abgemeldet. Wir entschuldigen uns für die Unannehmlichkeiten."), "autoPair": MessageLookupByLibrary.simpleMessage("Automatisch verbinden"), "autoPairDesc": MessageLookupByLibrary.simpleMessage( @@ -825,6 +825,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Zugriff gewähren"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Fotos in der Nähe gruppieren"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Wir tracken keine App-Installationen. Es würde uns jedoch helfen, wenn du uns mitteilst, wie du von uns erfahren hast!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1051,8 +1052,8 @@ class MessageLookup extends MessageLookupByLibrary { "Momentan werden keine Fotos gesichert"), "noPhotosFoundHere": MessageLookupByLibrary.simpleMessage("Keine Fotos gefunden"), - "noQuickLinksSelected": - MessageLookupByLibrary.simpleMessage("No quick links selected"), + "noQuickLinksSelected": MessageLookupByLibrary.simpleMessage( + "Keine schnellen Links ausgewählt"), "noRecoveryKey": MessageLookupByLibrary.simpleMessage( "Kein Wiederherstellungs-Schlüssel?"), "noRecoveryKeyNoDecryption": MessageLookupByLibrary.simpleMessage( @@ -1084,7 +1085,7 @@ class MessageLookup extends MessageLookupByLibrary { "openstreetmapContributors": MessageLookupByLibrary.simpleMessage("OpenStreetMap-Beitragende"), "optionalAsShortAsYouLike": MessageLookupByLibrary.simpleMessage( - "Bei Bedarf auch so kurz wie Sie wollen..."), + "Bei Bedarf auch so kurz wie du willst..."), "orPickAnExistingOne": MessageLookupByLibrary.simpleMessage( "Oder eine vorherige auswählen"), "pair": MessageLookupByLibrary.simpleMessage("Koppeln"), @@ -1159,7 +1160,7 @@ class MessageLookup extends MessageLookupByLibrary { "pleaseLoginAgain": MessageLookupByLibrary.simpleMessage("Bitte logge dich erneut ein"), "pleaseSelectQuickLinksToRemove": MessageLookupByLibrary.simpleMessage( - "Please select quick links to remove"), + "Bitte wähle die zu entfernenden schnellen Links"), "pleaseSendTheLogsTo": m42, "pleaseTryAgain": MessageLookupByLibrary.simpleMessage("Bitte versuche es erneut"), @@ -1267,7 +1268,7 @@ class MessageLookup extends MessageLookupByLibrary { "removePublicLink": MessageLookupByLibrary.simpleMessage("Öffentlichen Link entfernen"), "removePublicLinks": - MessageLookupByLibrary.simpleMessage("Remove public links"), + MessageLookupByLibrary.simpleMessage("Öffentliche Links entfernen"), "removeShareItemsWarning": MessageLookupByLibrary.simpleMessage( "Einige der Elemente, die du entfernst, wurden von anderen Nutzern hinzugefügt und du wirst den Zugriff auf sie verlieren"), "removeWithQuestionMark": @@ -1514,7 +1515,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Verbesserung vorschlagen"), "support": MessageLookupByLibrary.simpleMessage("Support"), "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "Um das Sperren beim Wischen zu aktivieren, richte bitte einen Gerätepasscode oder eine Bildschirmsperre in den Systemeinstellungen ein."), + "Um die Sperre für die Wischfunktion zu aktivieren, richte bitte einen Gerätepasscode oder eine Bildschirmsperre in den Systemeinstellungen ein."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Synchronisierung angehalten"), @@ -1572,7 +1573,7 @@ class MessageLookup extends MessageLookupByLibrary { "Dadurch wirst du von diesem Gerät abgemeldet!"), "thisWillRemovePublicLinksOfAllSelectedQuickLinks": MessageLookupByLibrary.simpleMessage( - "This will remove public links of all selected quick links."), + "Hiermit werden die öffentlichen Links aller ausgewählten schnellen Links entfernt."), "toEnableAppLockPleaseSetupDevicePasscodeOrScreen": MessageLookupByLibrary.simpleMessage( "Um die App-Sperre zu aktivieren, konfigurieren Sie bitte den Gerätepasscode oder die Bildschirmsperre in Ihren Systemeinstellungen."), @@ -1637,7 +1638,7 @@ class MessageLookup extends MessageLookupByLibrary { "useAsCover": MessageLookupByLibrary.simpleMessage("Als Titelbild festlegen"), "usePublicLinksForPeopleNotOnEnte": MessageLookupByLibrary.simpleMessage( - "Verwenden Sie öffentliche Links für Personen, die kein Ente-Konto haben"), + "Verwende öffentliche Links für Personen, die kein Ente-Konto haben"), "useRecoveryKey": MessageLookupByLibrary.simpleMessage( "Wiederherstellungs-Schlüssel verwenden"), "useSelectedPhoto": @@ -1717,12 +1718,12 @@ class MessageLookup extends MessageLookupByLibrary { "* Du kannst deinen Speicher maximal verdoppeln"), "youCanManageYourLinksInTheShareTab": MessageLookupByLibrary.simpleMessage( - "Sie können Ihre Links im \"Teilen\"-Tab verwalten."), + "Du kannst deine Links im \"Teilen\"-Tab verwalten."), "youCanTrySearchingForADifferentQuery": MessageLookupByLibrary.simpleMessage( "Sie können versuchen, nach einer anderen Abfrage suchen."), "youCannotDowngradeToThisPlan": MessageLookupByLibrary.simpleMessage( - "Sie können nicht auf diesen Tarif wechseln"), + "Du kannst nicht auf diesen Tarif wechseln"), "youCannotShareWithYourself": MessageLookupByLibrary.simpleMessage( "Du kannst nicht mit dir selbst teilen"), "youDontHaveAnyArchivedItems": MessageLookupByLibrary.simpleMessage( @@ -1733,11 +1734,11 @@ class MessageLookup extends MessageLookupByLibrary { "yourMap": MessageLookupByLibrary.simpleMessage("Deine Karte"), "yourPlanWasSuccessfullyDowngraded": MessageLookupByLibrary.simpleMessage( - "Ihr Tarif wurde erfolgreich heruntergestuft"), + "Dein Tarif wurde erfolgreich heruntergestuft"), "yourPlanWasSuccessfullyUpgraded": MessageLookupByLibrary.simpleMessage( - "Ihr Abo wurde erfolgreich aufgestuft"), + "Dein Abo wurde erfolgreich hochgestuft"), "yourPurchaseWasSuccessful": MessageLookupByLibrary.simpleMessage( - "Ihr Einkauf war erfolgreich!"), + "Dein Einkauf war erfolgreich"), "yourStorageDetailsCouldNotBeFetched": MessageLookupByLibrary.simpleMessage( "Details zum Speicherplatz konnten nicht abgerufen werden"), diff --git a/mobile/lib/generated/intl/messages_en.dart b/mobile/lib/generated/intl/messages_en.dart index 2f5c027183..6eafdbf79a 100644 --- a/mobile/lib/generated/intl/messages_en.dart +++ b/mobile/lib/generated/intl/messages_en.dart @@ -793,6 +793,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Grant permission"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Group nearby photos"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "We don\'t track app installs. It\'d help if you told us where you found us!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_es.dart b/mobile/lib/generated/intl/messages_es.dart index 4af7c3d044..25d102c0e6 100644 --- a/mobile/lib/generated/intl/messages_es.dart +++ b/mobile/lib/generated/intl/messages_es.dart @@ -827,6 +827,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Conceder permiso"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Agrupar fotos cercanas"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "No rastreamos las aplicaciones instaladas. ¡Nos ayudarías si nos dijeras dónde nos encontraste!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_fr.dart b/mobile/lib/generated/intl/messages_fr.dart index 0fb2c33ee4..3abb699e48 100644 --- a/mobile/lib/generated/intl/messages_fr.dart +++ b/mobile/lib/generated/intl/messages_fr.dart @@ -775,6 +775,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Accorder la permission"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Grouper les photos à proximité"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Nous ne suivons pas les installations d\'applications. Il serait utile que vous nous disiez comment vous nous avez trouvés !"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_it.dart b/mobile/lib/generated/intl/messages_it.dart index bcf137c66c..c978ee12d6 100644 --- a/mobile/lib/generated/intl/messages_it.dart +++ b/mobile/lib/generated/intl/messages_it.dart @@ -745,6 +745,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Concedi il permesso"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Raggruppa foto nelle vicinanze"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Non teniamo traccia del numero di installazioni dell\'app. Sarebbe utile se ci dicesse dove ci ha trovato!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_ko.dart b/mobile/lib/generated/intl/messages_ko.dart index 1c94d98677..8b30ee0b86 100644 --- a/mobile/lib/generated/intl/messages_ko.dart +++ b/mobile/lib/generated/intl/messages_ko.dart @@ -60,6 +60,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Face recognition"), "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hideContent": MessageLookupByLibrary.simpleMessage("Hide content"), "hideContentDescriptionAndroid": MessageLookupByLibrary.simpleMessage( "Hides app content in the app switcher and disables screenshots"), diff --git a/mobile/lib/generated/intl/messages_nl.dart b/mobile/lib/generated/intl/messages_nl.dart index 5c2cf3bb89..fd84335925 100644 --- a/mobile/lib/generated/intl/messages_nl.dart +++ b/mobile/lib/generated/intl/messages_nl.dart @@ -828,6 +828,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Toestemming verlenen"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Groep foto\'s in de buurt"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Wij gebruiken geen tracking. Het zou helpen als je ons vertelt waar je ons gevonden hebt!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_no.dart b/mobile/lib/generated/intl/messages_no.dart index d0297e1052..e737cd0591 100644 --- a/mobile/lib/generated/intl/messages_no.dart +++ b/mobile/lib/generated/intl/messages_no.dart @@ -78,6 +78,7 @@ class MessageLookup extends MessageLookupByLibrary { "feedback": MessageLookupByLibrary.simpleMessage("Tilbakemelding"), "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hideContent": MessageLookupByLibrary.simpleMessage("Hide content"), "hideContentDescriptionAndroid": MessageLookupByLibrary.simpleMessage( "Hides app content in the app switcher and disables screenshots"), diff --git a/mobile/lib/generated/intl/messages_pl.dart b/mobile/lib/generated/intl/messages_pl.dart index b5dcda8681..9d51285d0e 100644 --- a/mobile/lib/generated/intl/messages_pl.dart +++ b/mobile/lib/generated/intl/messages_pl.dart @@ -818,6 +818,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Przyznaj uprawnienie"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Grupuj pobliskie zdjęcia"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Nie śledzimy instalacji aplikacji. Pomogłyby nam, gdybyś powiedział/a nam, gdzie nas znalazłeś/aś!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1045,8 +1046,8 @@ class MessageLookup extends MessageLookupByLibrary { "W tej chwili nie wykonuje się kopii zapasowej zdjęć"), "noPhotosFoundHere": MessageLookupByLibrary.simpleMessage("Nie znaleziono tutaj zdjęć"), - "noQuickLinksSelected": - MessageLookupByLibrary.simpleMessage("No quick links selected"), + "noQuickLinksSelected": MessageLookupByLibrary.simpleMessage( + "Nie wybrano żadnych szybkich linków"), "noRecoveryKey": MessageLookupByLibrary.simpleMessage("Brak klucza odzyskiwania?"), "noRecoveryKeyNoDecryption": MessageLookupByLibrary.simpleMessage( @@ -1153,7 +1154,7 @@ class MessageLookup extends MessageLookupByLibrary { "pleaseLoginAgain": MessageLookupByLibrary.simpleMessage("Zaloguj się ponownie"), "pleaseSelectQuickLinksToRemove": MessageLookupByLibrary.simpleMessage( - "Please select quick links to remove"), + "Prosimy wybrać szybkie linki do usunięcia"), "pleaseSendTheLogsTo": m42, "pleaseTryAgain": MessageLookupByLibrary.simpleMessage("Spróbuj ponownie"), @@ -1257,7 +1258,7 @@ class MessageLookup extends MessageLookupByLibrary { "removePublicLink": MessageLookupByLibrary.simpleMessage("Usuń link publiczny"), "removePublicLinks": - MessageLookupByLibrary.simpleMessage("Remove public links"), + MessageLookupByLibrary.simpleMessage("Usuń linki publiczne"), "removeShareItemsWarning": MessageLookupByLibrary.simpleMessage( "Niektóre z usuwanych elementów zostały dodane przez inne osoby i utracisz do nich dostęp"), "removeWithQuestionMark": @@ -1562,7 +1563,7 @@ class MessageLookup extends MessageLookupByLibrary { "To wyloguje Cię z tego urządzenia!"), "thisWillRemovePublicLinksOfAllSelectedQuickLinks": MessageLookupByLibrary.simpleMessage( - "This will remove public links of all selected quick links."), + "Spowoduje to usunięcie publicznych linków wszystkich zaznaczonych szybkich linków."), "toEnableAppLockPleaseSetupDevicePasscodeOrScreen": MessageLookupByLibrary.simpleMessage( "Aby włączyć blokadę aplikacji, należy skonfigurować hasło urządzenia lub blokadę ekranu w ustawieniach systemu."), diff --git a/mobile/lib/generated/intl/messages_pt.dart b/mobile/lib/generated/intl/messages_pt.dart index 509de98a10..6e1e0d8947 100644 --- a/mobile/lib/generated/intl/messages_pt.dart +++ b/mobile/lib/generated/intl/messages_pt.dart @@ -817,6 +817,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Conceder permissão"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Agrupar fotos próximas"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Não rastreamos instalações do aplicativo. Seria útil se você nos contasse onde nos encontrou!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1041,8 +1042,8 @@ class MessageLookup extends MessageLookupByLibrary { "No momento não há backup de fotos sendo feito"), "noPhotosFoundHere": MessageLookupByLibrary.simpleMessage( "Nenhuma foto encontrada aqui"), - "noQuickLinksSelected": - MessageLookupByLibrary.simpleMessage("No quick links selected"), + "noQuickLinksSelected": MessageLookupByLibrary.simpleMessage( + "Nenhum link rápido selecionado"), "noRecoveryKey": MessageLookupByLibrary.simpleMessage( "Nenhuma chave de recuperação?"), "noRecoveryKeyNoDecryption": MessageLookupByLibrary.simpleMessage( @@ -1148,7 +1149,7 @@ class MessageLookup extends MessageLookupByLibrary { "pleaseLoginAgain": MessageLookupByLibrary.simpleMessage( "Por favor, inicie sessão novamente"), "pleaseSelectQuickLinksToRemove": MessageLookupByLibrary.simpleMessage( - "Please select quick links to remove"), + "Selecione links rápidos para remover"), "pleaseSendTheLogsTo": m42, "pleaseTryAgain": MessageLookupByLibrary.simpleMessage("Por favor, tente novamente"), @@ -1253,7 +1254,7 @@ class MessageLookup extends MessageLookupByLibrary { "removePublicLink": MessageLookupByLibrary.simpleMessage("Remover link público"), "removePublicLinks": - MessageLookupByLibrary.simpleMessage("Remove public links"), + MessageLookupByLibrary.simpleMessage("Remover link público"), "removeShareItemsWarning": MessageLookupByLibrary.simpleMessage( "Alguns dos itens que você está removendo foram adicionados por outras pessoas, e você perderá o acesso a eles"), "removeWithQuestionMark": @@ -1562,7 +1563,7 @@ class MessageLookup extends MessageLookupByLibrary { "Isso fará com que você saia deste dispositivo!"), "thisWillRemovePublicLinksOfAllSelectedQuickLinks": MessageLookupByLibrary.simpleMessage( - "This will remove public links of all selected quick links."), + "Isto removerá links públicos de todos os links rápidos selecionados."), "toEnableAppLockPleaseSetupDevicePasscodeOrScreen": MessageLookupByLibrary.simpleMessage( "Para ativar o bloqueio de app, por favor ative um método de autenticação nas configurações do sistema do seu dispositivo."), diff --git a/mobile/lib/generated/intl/messages_ru.dart b/mobile/lib/generated/intl/messages_ru.dart index e6fc6eaff7..08becfa5fe 100644 --- a/mobile/lib/generated/intl/messages_ru.dart +++ b/mobile/lib/generated/intl/messages_ru.dart @@ -811,6 +811,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("Предоставить разрешение"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Группировать фотографии рядом"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Будет полезно, если вы укажете, где нашли нас, так как мы не отслеживаем установки приложения!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_tr.dart b/mobile/lib/generated/intl/messages_tr.dart index dbfa3d4f83..d4821c86b5 100644 --- a/mobile/lib/generated/intl/messages_tr.dart +++ b/mobile/lib/generated/intl/messages_tr.dart @@ -813,6 +813,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("İzinleri değiştir"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Yakındaki fotoğrafları gruplandır"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Biz uygulama kurulumlarını takip etmiyoruz. Bizi nereden duyduğunuzdan bahsetmeniz bize çok yardımcı olacak!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_zh.dart b/mobile/lib/generated/intl/messages_zh.dart index 9a4a812fe1..164d5b87f9 100644 --- a/mobile/lib/generated/intl/messages_zh.dart +++ b/mobile/lib/generated/intl/messages_zh.dart @@ -367,7 +367,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("请确保您的设备与电视处于同一网络。"), "castIPMismatchTitle": MessageLookupByLibrary.simpleMessage("投放相册失败"), "castInstruction": MessageLookupByLibrary.simpleMessage( - "在您要配对的设备上访问 cast.ente.io。\n输入下面的代码即可在电视上播放相册。"), + "在您要配对的设备上访问 cast.ente.io。\n在下框中输入代码即可在电视上播放相册。"), "centerPoint": MessageLookupByLibrary.simpleMessage("中心点"), "changeEmail": MessageLookupByLibrary.simpleMessage("修改邮箱"), "changeLocationOfSelectedItems": @@ -669,6 +669,7 @@ class MessageLookup extends MessageLookupByLibrary { MessageLookupByLibrary.simpleMessage("请在手机“设置”中授权软件访问所有照片"), "grantPermission": MessageLookupByLibrary.simpleMessage("授予权限"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("将附近的照片分组"), + "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "我们不跟踪应用程序安装情况。如果您告诉我们您是在哪里找到我们的，将会有所帮助！"), "hearUsWhereTitle": @@ -857,8 +858,7 @@ class MessageLookup extends MessageLookupByLibrary { "noPhotosAreBeingBackedUpRightNow": MessageLookupByLibrary.simpleMessage("目前没有照片正在备份"), "noPhotosFoundHere": MessageLookupByLibrary.simpleMessage("这里没有找到照片"), - "noQuickLinksSelected": - MessageLookupByLibrary.simpleMessage("No quick links selected"), + "noQuickLinksSelected": MessageLookupByLibrary.simpleMessage("未选择快速链接"), "noRecoveryKey": MessageLookupByLibrary.simpleMessage("没有恢复密钥吗？"), "noRecoveryKeyNoDecryption": MessageLookupByLibrary.simpleMessage( "由于我们端到端加密协议的性质，如果没有您的密码或恢复密钥，您的数据将无法解密"), @@ -941,8 +941,8 @@ class MessageLookup extends MessageLookupByLibrary { "pleaseEmailUsAt": m41, "pleaseGrantPermissions": MessageLookupByLibrary.simpleMessage("请授予权限"), "pleaseLoginAgain": MessageLookupByLibrary.simpleMessage("请重新登录"), - "pleaseSelectQuickLinksToRemove": MessageLookupByLibrary.simpleMessage( - "Please select quick links to remove"), + "pleaseSelectQuickLinksToRemove": + MessageLookupByLibrary.simpleMessage("请选择要删除的快速链接"), "pleaseSendTheLogsTo": m42, "pleaseTryAgain": MessageLookupByLibrary.simpleMessage("请重试"), "pleaseVerifyTheCodeYouHaveEntered": @@ -1020,8 +1020,7 @@ class MessageLookup extends MessageLookupByLibrary { "removeParticipantBody": m45, "removePersonLabel": MessageLookupByLibrary.simpleMessage("移除人物标签"), "removePublicLink": MessageLookupByLibrary.simpleMessage("删除公开链接"), - "removePublicLinks": - MessageLookupByLibrary.simpleMessage("Remove public links"), + "removePublicLinks": MessageLookupByLibrary.simpleMessage("删除公开链接"), "removeShareItemsWarning": MessageLookupByLibrary.simpleMessage("您要删除的某些项目是由其他人添加的，您将无法访问它们"), "removeWithQuestionMark": MessageLookupByLibrary.simpleMessage("要移除吗?"), @@ -1261,8 +1260,7 @@ class MessageLookup extends MessageLookupByLibrary { "thisWillLogYouOutOfThisDevice": MessageLookupByLibrary.simpleMessage("这将使您在此设备上退出登录！"), "thisWillRemovePublicLinksOfAllSelectedQuickLinks": - MessageLookupByLibrary.simpleMessage( - "This will remove public links of all selected quick links."), + MessageLookupByLibrary.simpleMessage("这将删除所有选定的快速链接的公共链接。"), "toEnableAppLockPleaseSetupDevicePasscodeOrScreen": MessageLookupByLibrary.simpleMessage("要启用应用锁，请在系统设置中设置设备密码或屏幕锁定。"), "toHideAPhotoOrVideo": MessageLookupByLibrary.simpleMessage("隐藏照片或视频"), diff --git a/mobile/lib/generated/l10n.dart b/mobile/lib/generated/l10n.dart index 5f3d7d8494..7add93a337 100644 --- a/mobile/lib/generated/l10n.dart +++ b/mobile/lib/generated/l10n.dart @@ -9244,6 +9244,16 @@ class S { args: [], ); } + + /// `Guest view` + String get guestView { + return Intl.message( + 'Guest view', + name: 'guestView', + desc: '', + args: [], + ); + } } class AppLocalizationDelegate extends LocalizationsDelegate { diff --git a/mobile/lib/l10n/intl_cs.arb b/mobile/lib/l10n/intl_cs.arb index 1df0bb4639..2f2942c442 100644 --- a/mobile/lib/l10n/intl_cs.arb +++ b/mobile/lib/l10n/intl_cs.arb @@ -52,5 +52,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_de.arb b/mobile/lib/l10n/intl_de.arb index c65d34b6d5..57f30fa51e 100644 --- a/mobile/lib/l10n/intl_de.arb +++ b/mobile/lib/l10n/intl_de.arb @@ -1290,5 +1290,6 @@ "noQuickLinksSelected": "Keine schnellen Links ausgewählt", "pleaseSelectQuickLinksToRemove": "Bitte wähle die zu entfernenden schnellen Links", "removePublicLinks": "Öffentliche Links entfernen", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Hiermit werden die öffentlichen Links aller ausgewählten schnellen Links entfernt." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Hiermit werden die öffentlichen Links aller ausgewählten schnellen Links entfernt.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index 81c5a8b427..e29bf9a14c 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -1290,5 +1290,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_es.arb b/mobile/lib/l10n/intl_es.arb index bdf5eefbb9..280d87d0de 100644 --- a/mobile/lib/l10n/intl_es.arb +++ b/mobile/lib/l10n/intl_es.arb @@ -1276,5 +1276,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_fr.arb b/mobile/lib/l10n/intl_fr.arb index b8a0e7eb02..7e7c87caf2 100644 --- a/mobile/lib/l10n/intl_fr.arb +++ b/mobile/lib/l10n/intl_fr.arb @@ -1193,5 +1193,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_it.arb b/mobile/lib/l10n/intl_it.arb index acfbd648ad..568c500b3e 100644 --- a/mobile/lib/l10n/intl_it.arb +++ b/mobile/lib/l10n/intl_it.arb @@ -1155,5 +1155,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_ko.arb b/mobile/lib/l10n/intl_ko.arb index 1df0bb4639..2f2942c442 100644 --- a/mobile/lib/l10n/intl_ko.arb +++ b/mobile/lib/l10n/intl_ko.arb @@ -52,5 +52,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_nl.arb b/mobile/lib/l10n/intl_nl.arb index cdc23ddb0d..b7778c513e 100644 --- a/mobile/lib/l10n/intl_nl.arb +++ b/mobile/lib/l10n/intl_nl.arb @@ -1290,5 +1290,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_no.arb b/mobile/lib/l10n/intl_no.arb index 023c3ba216..9c1aaeda09 100644 --- a/mobile/lib/l10n/intl_no.arb +++ b/mobile/lib/l10n/intl_no.arb @@ -66,5 +66,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_pl.arb b/mobile/lib/l10n/intl_pl.arb index 63e3c47879..a13d440446 100644 --- a/mobile/lib/l10n/intl_pl.arb +++ b/mobile/lib/l10n/intl_pl.arb @@ -1290,5 +1290,6 @@ "noQuickLinksSelected": "Nie wybrano żadnych szybkich linków", "pleaseSelectQuickLinksToRemove": "Prosimy wybrać szybkie linki do usunięcia", "removePublicLinks": "Usuń linki publiczne", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Spowoduje to usunięcie publicznych linków wszystkich zaznaczonych szybkich linków." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Spowoduje to usunięcie publicznych linków wszystkich zaznaczonych szybkich linków.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_pt.arb b/mobile/lib/l10n/intl_pt.arb index 28ac1fbbd8..7c9252873b 100644 --- a/mobile/lib/l10n/intl_pt.arb +++ b/mobile/lib/l10n/intl_pt.arb @@ -1290,5 +1290,6 @@ "noQuickLinksSelected": "Nenhum link rápido selecionado", "pleaseSelectQuickLinksToRemove": "Selecione links rápidos para remover", "removePublicLinks": "Remover link público", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Isto removerá links públicos de todos os links rápidos selecionados." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Isto removerá links públicos de todos os links rápidos selecionados.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_ru.arb b/mobile/lib/l10n/intl_ru.arb index c0044e3ee4..428cdea90d 100644 --- a/mobile/lib/l10n/intl_ru.arb +++ b/mobile/lib/l10n/intl_ru.arb @@ -1275,5 +1275,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_tr.arb b/mobile/lib/l10n/intl_tr.arb index 86b8dc2777..ff66a5dfd1 100644 --- a/mobile/lib/l10n/intl_tr.arb +++ b/mobile/lib/l10n/intl_tr.arb @@ -1287,5 +1287,6 @@ "noQuickLinksSelected": "No quick links selected", "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links." + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_zh.arb b/mobile/lib/l10n/intl_zh.arb index e51872b4a5..fac2a8f82b 100644 --- a/mobile/lib/l10n/intl_zh.arb +++ b/mobile/lib/l10n/intl_zh.arb @@ -1290,5 +1290,6 @@ "noQuickLinksSelected": "未选择快速链接", "pleaseSelectQuickLinksToRemove": "请选择要删除的快速链接", "removePublicLinks": "删除公开链接", - "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "这将删除所有选定的快速链接的公共链接。" + "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "这将删除所有选定的快速链接的公共链接。", + "guestView": "Guest view" } \ No newline at end of file diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index 1ab3d0fb5b..c5af5deab4 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -275,7 +275,7 @@ class _FileSelectionActionsWidgetState items.add( SelectionActionButton( icon: Icons.people_outline_rounded, - labelText: "Guest view", + labelText: S.of(context).guestView, onTap: _onGuestViewClick, ), ); diff --git a/mobile/lib/ui/viewer/file/file_app_bar.dart b/mobile/lib/ui/viewer/file/file_app_bar.dart index 6120ffb5df..e9221b2f72 100644 --- a/mobile/lib/ui/viewer/file/file_app_bar.dart +++ b/mobile/lib/ui/viewer/file/file_app_bar.dart @@ -305,7 +305,7 @@ class FileAppBarState extends State { const Padding( padding: EdgeInsets.all(8), ), - const Text("Guest view"), + Text(S.of(context).guestView), ], ), ), From 29d7403cda710fb086153d434fb526b56398c180 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 12:43:56 +0530 Subject: [PATCH 112/211] Non-trivial ones --- web/packages/new/photos/services/ml/cluster-new.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index 3a615471d4..298eafc26b 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -78,7 +78,7 @@ export const clusterFaces = (faceIndexes: FaceIndex[]) => { const faces = [...faceIDAndEmbeddings(faceIndexes)]; - const clusters: FaceCluster[] = []; + let clusters: FaceCluster[] = []; const clusterIndexByFaceID = new Map(); for (const [i, { faceID, embedding }] of faces.entries()) { // Find the nearest neighbour from among the faces we have already seen. @@ -118,6 +118,8 @@ export const clusterFaces = (faceIndexes: FaceIndex[]) => { } } + clusters = clusters.filter(({ faceIDs }) => faceIDs.length > 1); + log.debug(() => ["ml/cluster", { faces, clusters, clusterIndexByFaceID }]); log.debug( () => From 7834662340a1f2ff1a806480862f4cf677c1466c Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 9 Aug 2024 13:08:46 +0530 Subject: [PATCH 113/211] [server] Clean up --- server/cmd/museum/main.go | 2 +- server/pkg/controller/filedata/controller.go | 14 ++------------ 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index f74b004421..7aa2f41c1f 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -241,7 +241,7 @@ func main() { } accessCtrl := access.NewAccessController(collectionRepo, fileRepo) - fileDataCtrl := filedata.New(fileDataRepo, accessCtrl, objectCleanupController, s3Config, queueRepo, taskLockingRepo, fileRepo, collectionRepo, hostName) + fileDataCtrl := filedata.New(fileDataRepo, accessCtrl, objectCleanupController, s3Config, fileRepo, collectionRepo) fileController := &controller.FileController{ FileRepo: fileRepo, diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index afdd895854..afd0a73a6f 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -48,14 +48,9 @@ type Controller struct { AccessCtrl access.Controller ObjectCleanupController *controller.ObjectCleanupController S3Config *s3config.S3Config - QueueRepo *repo.QueueRepository - TaskLockingRepo *repo.TaskLockRepository FileRepo *repo.FileRepository CollectionRepo *repo.CollectionRepository - HostName string - cleanupCronRunning bool downloadManagerCache map[string]*s3manager.Downloader - // for downloading objects from s3 for replication workerURL string } @@ -64,11 +59,8 @@ func New(repo *fileDataRepo.Repository, accessCtrl access.Controller, objectCleanupController *controller.ObjectCleanupController, s3Config *s3config.S3Config, - queueRepo *repo.QueueRepository, - taskLockingRepo *repo.TaskLockRepository, fileRepo *repo.FileRepository, - collectionRepo *repo.CollectionRepository, - hostName string) *Controller { + collectionRepo *repo.CollectionRepository) *Controller { embeddingDcs := []string{s3Config.GetHotBackblazeDC(), s3Config.GetHotWasabiDC(), s3Config.GetWasabiDerivedDC(), s3Config.GetDerivedStorageDataCenter(), "b5"} cache := make(map[string]*s3manager.Downloader, len(embeddingDcs)) for i := range embeddingDcs { @@ -80,11 +72,8 @@ func New(repo *fileDataRepo.Repository, AccessCtrl: accessCtrl, ObjectCleanupController: objectCleanupController, S3Config: s3Config, - QueueRepo: queueRepo, - TaskLockingRepo: taskLockingRepo, FileRepo: fileRepo, CollectionRepo: collectionRepo, - HostName: hostName, downloadManagerCache: cache, } } @@ -261,6 +250,7 @@ func (c *Controller) _validateGetFilesData(ctx *gin.Context, userID int64, req f }); err != nil { return stacktrace.Propagate(err, "User does not own some file(s)") } + return nil } From 5fa719f3e9e51c604e6206e63ad8f5b9edaa8007 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 13:26:32 +0530 Subject: [PATCH 114/211] Sketch --- web/apps/photos/src/services/entityService.ts | 2 +- web/packages/new/photos/services/entity.ts | 60 +++++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 web/packages/new/photos/services/entity.ts diff --git a/web/apps/photos/src/services/entityService.ts b/web/apps/photos/src/services/entityService.ts index 401819b110..1aeba5ff44 100644 --- a/web/apps/photos/src/services/entityService.ts +++ b/web/apps/photos/src/services/entityService.ts @@ -25,7 +25,7 @@ import { getLatestVersionEntities } from "utils/entity"; * * These diff API calls return all items whose updated at is greater * (non-inclusive) than the timestamp we provide. So there is no mechanism for - * pagination of items which have the same exact updated at. + * pagination of items which have the exact same updated at. * * Conceptually, it may happen that there are more items than the limit we've * provided, but there are practical safeguards. diff --git a/web/packages/new/photos/services/entity.ts b/web/packages/new/photos/services/entity.ts new file mode 100644 index 0000000000..363417293f --- /dev/null +++ b/web/packages/new/photos/services/entity.ts @@ -0,0 +1,60 @@ +/** + * Entities are predefined lists of otherwise arbitrary data that the user can + * store for their account. + * + * e.g. location tags, people in their photos. + */ +export type EntityType = + /** + * A new version of the Person entity where the data is gzipped before + * encryption. + */ + "person_v2"; + +/** + * The maximum number of items to fetch in a single diff + * + * [Note: Limit of returned items in /diff requests] + * + * The various GET /diff API methods, which tell the client what all has changed + * since a timestamp (provided by the client) take a limit parameter. + * + * These diff API calls return all items whose updated at is greater + * (non-inclusive) than the timestamp we provide. So there is no mechanism for + * pagination of items which have the exact same updated at. + * + * Conceptually, it may happen that there are more items than the limit we've + * provided, but there are practical safeguards. + * + * For file diff, the limit is advisory, and remote may return less, equal or + * more items than the provided limit. The scenario where it returns more is + * when more files than the limit have the same updated at. Theoretically it + * would make the diff response unbounded, however in practice file + * modifications themselves are all batched. Even if the user were to select all + * the files in their library and updates them all in one go in the UI, their + * client app is required to use batched API calls to make those updates, and + * each of those batches would get distinct updated at. + */ +const defaultDiffLimit = 500; + +/** + * Fetch all entities of the given type that have been created or updated since + * the given time. + * + * For each batch of fetched entities, call a provided function that can ingest + * them. This function is also provided the latest timestamp from amongst all of + * the entities fetched so far, which the caller can persist if needed to resume + * future diffs from the current checkpoint. + */ +export const entityDiff = async () => { + const sinceTime = 0; + const type: EntityType = "person_v2"; + const limit = defaultDiffLimit; + const params = new URLSearchParams({ type, sinceTime, limit }); + // const url = await apiURL(`/user-entity/entity/diff`); + // const res = await fetch(`${url}?${params.toString()}`, { + // headers: await authenticatedRequestHeaders(), + // }); + // ensureOk(res); + // }; +}; From 777ce3f4a8e1f6e31d27b8d0d8101b67cc259f3e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 13:38:35 +0530 Subject: [PATCH 115/211] Sketch --- web/packages/new/photos/services/entity.ts | 37 +++++++++++++++------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/web/packages/new/photos/services/entity.ts b/web/packages/new/photos/services/entity.ts index 363417293f..dce7c3fa6c 100644 --- a/web/packages/new/photos/services/entity.ts +++ b/web/packages/new/photos/services/entity.ts @@ -1,3 +1,6 @@ +import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; +import { apiURL } from "@/base/origins"; + /** * Entities are predefined lists of otherwise arbitrary data that the user can * store for their account. @@ -44,17 +47,27 @@ const defaultDiffLimit = 500; * For each batch of fetched entities, call a provided function that can ingest * them. This function is also provided the latest timestamp from amongst all of * the entities fetched so far, which the caller can persist if needed to resume - * future diffs from the current checkpoint. + * future diffs fetches from the current checkpoint. + * + * @param type The type of the entities to fetch. + * + * @param sinceTime Epoch milliseconds. This is used to ask remote to provide us + * only entities whose {@link updatedAt} is more than the given value. Set this + * to zero to start from the beginning. */ -export const entityDiff = async () => { - const sinceTime = 0; - const type: EntityType = "person_v2"; - const limit = defaultDiffLimit; - const params = new URLSearchParams({ type, sinceTime, limit }); - // const url = await apiURL(`/user-entity/entity/diff`); - // const res = await fetch(`${url}?${params.toString()}`, { - // headers: await authenticatedRequestHeaders(), - // }); - // ensureOk(res); - // }; +export const entityDiff = async ( + type: EntityType, + sinceTime: number, + onFetch: (entities: unknown[], latestUpdatedAt: number) => Promise, +) => { + const params = new URLSearchParams({ + type, + sinceTime: sinceTime.toString(), + limit: defaultDiffLimit.toString(), + }); + const url = await apiURL(`/user-entity/entity/diff`); + const res = await fetch(`${url}?${params.toString()}`, { + headers: await authenticatedRequestHeaders(), + }); + ensureOk(res); }; From 0412c37bf51fe4178e48969d3cd95feec2d6fd5c Mon Sep 17 00:00:00 2001 From: Vishnu Mohandas Date: Fri, 9 Aug 2024 14:14:52 +0530 Subject: [PATCH 116/211] Update index.md --- docs/docs/photos/migration/from-google-photos/index.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/docs/photos/migration/from-google-photos/index.md b/docs/docs/photos/migration/from-google-photos/index.md index feaaf7784a..772908576e 100644 --- a/docs/docs/photos/migration/from-google-photos/index.md +++ b/docs/docs/photos/migration/from-google-photos/index.md @@ -59,6 +59,5 @@ If you run into any issues during this migration, please reach out to > JSON files and stich them together with corresponding files. However, one case > this will not work is when Google has split the export into multiple parts, > and did not put the JSON file associated with an image in the same exported -> zip. - ->So the best move is to unzip all of the items into a single root folder, and drop that folder into our desktop app. That way we have the complete picture, and can stitch together metadata with the correct files. \ No newline at end of file +> zip. So the best move is to unzip all of the items into a single folder, and +> to drop that folder into our desktop app. From 1c84b326087c9a279d9f2d8bd3a5e4c276b98803 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 14:22:26 +0530 Subject: [PATCH 117/211] Specifize --- .../new/photos/services/{entity.ts => user-entity.ts} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename web/packages/new/photos/services/{entity.ts => user-entity.ts} (91%) diff --git a/web/packages/new/photos/services/entity.ts b/web/packages/new/photos/services/user-entity.ts similarity index 91% rename from web/packages/new/photos/services/entity.ts rename to web/packages/new/photos/services/user-entity.ts index dce7c3fa6c..e08e3874e8 100644 --- a/web/packages/new/photos/services/entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -2,8 +2,8 @@ import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; /** - * Entities are predefined lists of otherwise arbitrary data that the user can - * store for their account. + * User entities are predefined lists of otherwise arbitrary data that the user + * can store for their account. * * e.g. location tags, people in their photos. */ @@ -41,8 +41,8 @@ export type EntityType = const defaultDiffLimit = 500; /** - * Fetch all entities of the given type that have been created or updated since - * the given time. + * Fetch all user entities of the given type that have been created or updated + * since the given time. * * For each batch of fetched entities, call a provided function that can ingest * them. This function is also provided the latest timestamp from amongst all of @@ -55,7 +55,7 @@ const defaultDiffLimit = 500; * only entities whose {@link updatedAt} is more than the given value. Set this * to zero to start from the beginning. */ -export const entityDiff = async ( +export const userEntityDiff = async ( type: EntityType, sinceTime: number, onFetch: (entities: unknown[], latestUpdatedAt: number) => Promise, From 8f167d81fc62d656bd538119f1b1fecb7da310df Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 14:36:27 +0530 Subject: [PATCH 118/211] Types --- .../new/photos/services/user-entity.ts | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index e08e3874e8..9b3335952e 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -1,5 +1,6 @@ import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; +import { z } from "zod"; /** * User entities are predefined lists of otherwise arbitrary data that the user @@ -40,26 +41,53 @@ export type EntityType = */ const defaultDiffLimit = 500; +/** + * A generic user entity. + * + * This is an intermediate step, usually what we really want is a version + * of this with the {@link data} parsed to the specific type of JSON object + * expected to be associated with this entity type. + */ +interface UserEntity { + /** + * Arbitrary data associated with the entity. The format of this data is + * specific to each entity type. + * + * This will not be present for entities that have been deleted on remote. + */ + data: Uint8Array | undefined; + /** + * Epoch microseconds denoting when this entity was created or last updated. + */ + updatedAt: number; +} + +const RemoteUserEntity = z.object({ + /** Base64 string containing the encrypted contents of the entity. */ + encryptedData: z.string(), + /** Base64 string containing the decryption header. */ + header: z.string(), + isDeleted: z.boolean(), + updatedAt: z.number(), +} + /** * Fetch all user entities of the given type that have been created or updated * since the given time. * - * For each batch of fetched entities, call a provided function that can ingest - * them. This function is also provided the latest timestamp from amongst all of - * the entities fetched so far, which the caller can persist if needed to resume - * future diffs fetches from the current checkpoint. - * * @param type The type of the entities to fetch. * * @param sinceTime Epoch milliseconds. This is used to ask remote to provide us * only entities whose {@link updatedAt} is more than the given value. Set this * to zero to start from the beginning. + * + * @param entityKey The key to use for decrypting the encrypted contents of the + * user entity. */ export const userEntityDiff = async ( type: EntityType, sinceTime: number, - onFetch: (entities: unknown[], latestUpdatedAt: number) => Promise, -) => { +): Promise => { const params = new URLSearchParams({ type, sinceTime: sinceTime.toString(), From e5eb9fee7ad5a70b6f9bafc64558e65c1fa83952 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 14:46:22 +0530 Subject: [PATCH 119/211] [docs] Clarifications --- .../docs/photos/troubleshooting/thumbnails.md | 24 ++++++++++--------- .../self-hosting/guides/configuring-s3.md | 7 ++++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/docs/photos/troubleshooting/thumbnails.md b/docs/docs/photos/troubleshooting/thumbnails.md index 81c8ff8227..f26319c2e5 100644 --- a/docs/docs/photos/troubleshooting/thumbnails.md +++ b/docs/docs/photos/troubleshooting/thumbnails.md @@ -23,10 +23,12 @@ canvas. ## Desktop The only known case where thumbnails might be missing on desktop is when -uploading **videos** during a Google Takeout or folder sync on **Intel macOS** -machines. This is because the bundled ffmpeg that we use does not work with -Rosetta. For images, we are able to fallback to other mechanisms for generating -the thumbnails, but for videos because of their potentially huge size, the app +uploading **videos** during a Google Takeout or watched folder sync on **Intel +macOS** machines. This is because the bundled ffmpeg that we use does not work +on Intel machines. + +For images, we are able to fallback to other mechanisms for generating the +thumbnails, but for videos because of their potentially huge size, the app doesn't try the fallback to avoid running out of memory. In such cases, you will need to use the following workaround: @@ -39,10 +41,11 @@ In such cases, you will need to use the following workaround: 2. Copy or symlink it to `/Applications/ente.app/Contents/Resources/app.asar.unpacked/node_modules/ffmpeg-static/ffmpeg`. -Even without the workaround, thumbnail generation during video uploads via the -normal folder selection or drag and drop will work fine (since in this case we -have access to the video's data directly without reading it from a zip and can -thus use the fallback). +Alternatively, you can drag and drop the videos. Even without the above +workaround, thumbnail generation during video uploads via the normal folder +selection or drag and drop will work fine, since in those case we have access to +the video's data directly without reading it from a zip and can thus use the +fallback. ## Regenerating thumbnails @@ -50,6 +53,5 @@ There is currently no functionality to regenerate thumbnails in the above cases. You will need to upload the affected files again. Ente skips over files that have already been uploaded, so you can drag and drop -the original folder or zip again after removing the files without thumbnails -(and fixing the issue on web or adding the workaround on Intel macOS), and it'll -only upload the files that are necessary. +the original folder or zip again after removing the files without thumbnails, +and it'll only upload the files that are necessary. diff --git a/docs/docs/self-hosting/guides/configuring-s3.md b/docs/docs/self-hosting/guides/configuring-s3.md index 56e922f02d..a5b9eccc8a 100644 --- a/docs/docs/self-hosting/guides/configuring-s3.md +++ b/docs/docs/self-hosting/guides/configuring-s3.md @@ -45,8 +45,11 @@ By default, you only need to configure the endpoint for the first bucket. > instance uses these to perform replication. > > However, in a self hosted setup replication is off by default (you can turn it -> on if you want). When replication is turned off, only the first bucket is -> used, and you can remove the other two if you wish or just ignore them. +> on if you want). When replication is turned off, only the first bucket (it +> must be named `b2-eu-cen`) is used, and you can ignore the other two. Use the +> `hot_bucket` option if you'd like to set one of the other predefined buckets +> as the "first" bucket. + The `endpoint` for the first bucket in the starter `credentials.yaml` is `localhost:3200`. The way this works then is that both museum (`2`) and minio From 286517338ef284d7e6fc981f125dc7db7dbc1b02 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 14:47:23 +0530 Subject: [PATCH 120/211] pretty --- docs/docs/self-hosting/guides/configuring-s3.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/docs/self-hosting/guides/configuring-s3.md b/docs/docs/self-hosting/guides/configuring-s3.md index a5b9eccc8a..3ab3828dae 100644 --- a/docs/docs/self-hosting/guides/configuring-s3.md +++ b/docs/docs/self-hosting/guides/configuring-s3.md @@ -50,7 +50,6 @@ By default, you only need to configure the endpoint for the first bucket. > `hot_bucket` option if you'd like to set one of the other predefined buckets > as the "first" bucket. - The `endpoint` for the first bucket in the starter `credentials.yaml` is `localhost:3200`. The way this works then is that both museum (`2`) and minio (`3`) are running within the same Docker compose cluster, so are able to reach From 980ff741ba35d048adb60f45ce7c96d19ae6d0f5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 14:48:08 +0530 Subject: [PATCH 121/211] yarn pretty --- docs/docs/auth/faq/privacy-disclosure/index.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/docs/auth/faq/privacy-disclosure/index.md b/docs/docs/auth/faq/privacy-disclosure/index.md index 5b1a990836..7ed0bc05a8 100644 --- a/docs/docs/auth/faq/privacy-disclosure/index.md +++ b/docs/docs/auth/faq/privacy-disclosure/index.md @@ -18,29 +18,33 @@ AppStore](appstore-privacy-disclosure.png){width=620px} ## Data Linked to You > [!NOTE] -> +> > Only if you choose to create an account to backup your codes are the following > details collected. ### Contact Info + This is your email address, used for account creation and communication. ### User Content + This are your 2FA secrets, end-to-end encrypted with a key that only you have access to. ### Identifiers + This is your user ID generated by our server during sign up. ## Data Not Linked to You > [!NOTE] -> +> > Only if you opt-in to **Crash reporting** are the following details collected. ### Diagnostics -These are anonymized error reports and other diagnostics data that make it easier -for us to detect and fix any issues. + +These are anonymized error reports and other diagnostics data that make it +easier for us to detect and fix any issues. --- @@ -48,5 +52,5 @@ for us to detect and fix any issues. Ente Auth collects no data by default. -For more details, please refer to our [full privacy -policy](https://ente.io/privacy). +For more details, please refer to our +[full privacy policy](https://ente.io/privacy). From 46082796d7c44bbe90230387b53be800c9e48c08 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 15:36:15 +0530 Subject: [PATCH 122/211] Decrypt --- web/packages/base/crypto/ente.ts | 25 ++++++++++++++---- .../new/photos/services/user-entity.ts | 26 ++++++++++++++++--- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index 017c38aa2c..f1dedb94e0 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -115,7 +115,7 @@ export const encryptMetadata = async (metadata: unknown, keyB64: string) => { * * This is the sibling of {@link encryptAssociatedData}. * - * See {@link decryptChaChaOneShot2} for the implementation details. + * See {@link decryptChaChaOneShot} for the implementation details. * * @param encryptedData A {@link Uint8Array} containing the bytes to decrypt. * @@ -166,7 +166,7 @@ export const decryptFileEmbedding = async ( * Decrypt the metadata associated with an Ente object (file, collection or * entity) using the object's key. * - * This is the sibling of {@link decryptMetadata}. + * This is the sibling of {@link encryptMetadata}. * * @param encryptedDataB64 base64 encoded string containing the encrypted data. * @@ -180,7 +180,6 @@ export const decryptFileEmbedding = async ( * @returns The decrypted JSON value. Since TypeScript does not have a native * JSON type, we need to return it as an `unknown`. */ - export const decryptMetadata = async ( encryptedDataB64: string, decryptionHeaderB64: string, @@ -188,10 +187,26 @@ export const decryptMetadata = async ( ) => JSON.parse( new TextDecoder().decode( - await decryptAssociatedData( - await libsodium.fromB64(encryptedDataB64), + await decryptMetadataBytes( + encryptedDataB64, decryptionHeaderB64, keyB64, ), ), ) as unknown; + +/** + * A variant of {@link decryptMetadata} that does not attempt to parse the + * decrypted data as a JSON string and instead just returns the raw decrypted + * bytes that we got. + */ +export const decryptMetadataBytes = async ( + encryptedDataB64: string, + decryptionHeaderB64: string, + keyB64: string, +) => + await decryptAssociatedData( + await libsodium.fromB64(encryptedDataB64), + decryptionHeaderB64, + keyB64, + ); diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 9b3335952e..939ed9d4e9 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -1,3 +1,4 @@ +import { decryptMetadataBytes } from "@/base/crypto/ente"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; import { z } from "zod"; @@ -69,7 +70,7 @@ const RemoteUserEntity = z.object({ header: z.string(), isDeleted: z.boolean(), updatedAt: z.number(), -} +}); /** * Fetch all user entities of the given type that have been created or updated @@ -81,13 +82,17 @@ const RemoteUserEntity = z.object({ * only entities whose {@link updatedAt} is more than the given value. Set this * to zero to start from the beginning. * - * @param entityKey The key to use for decrypting the encrypted contents of the - * user entity. + * @param entityKeyB64 The base64 encoded key to use for decrypting the + * encrypted contents of the user entity. */ export const userEntityDiff = async ( type: EntityType, sinceTime: number, -): Promise => { + entityKeyB64: string, +): Promise => { + const decrypt = (dataB64: string, headerB64: string) => + decryptMetadataBytes(dataB64, headerB64, entityKeyB64); + const params = new URLSearchParams({ type, sinceTime: sinceTime.toString(), @@ -98,4 +103,17 @@ export const userEntityDiff = async ( headers: await authenticatedRequestHeaders(), }); ensureOk(res); + const entities = z + .object({ diff: z.array(RemoteUserEntity) }) + .parse(await res.json()).diff; + return Promise.all( + entities.map( + async ({ encryptedData, header, isDeleted, updatedAt }) => ({ + data: isDeleted + ? undefined + : await decrypt(encryptedData, header), + updatedAt, + }), + ), + ); }; From 7bde0dea8ca1f822c50b791067c9898618af3343 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 15:48:30 +0530 Subject: [PATCH 123/211] test harness --- web/apps/photos/src/services/entityService.ts | 3 ++- web/apps/photos/src/services/searchService.ts | 14 +++++++++++++- web/packages/new/photos/services/ml/index.ts | 10 ++++++++-- web/packages/new/photos/services/user-entity.ts | 15 +++++++++++++++ 4 files changed, 38 insertions(+), 4 deletions(-) diff --git a/web/apps/photos/src/services/entityService.ts b/web/apps/photos/src/services/entityService.ts index 1aeba5ff44..be8a1e3a7f 100644 --- a/web/apps/photos/src/services/entityService.ts +++ b/web/apps/photos/src/services/entityService.ts @@ -71,7 +71,8 @@ const getCachedEntityKey = async (type: EntityType) => { return entityKey; }; -const getEntityKey = async (type: EntityType) => { +// TODO: unexport +export const getEntityKey = async (type: EntityType) => { try { const entityKey = await getCachedEntityKey(type); if (entityKey) { diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index c697c34575..2bb7d304cb 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -7,8 +7,10 @@ import { isMLSupported, mlStatusSnapshot, wipCluster, + wipClusterEnable, } from "@/new/photos/services/ml"; import type { Person } from "@/new/photos/services/ml/people"; +import { personDiff } from "@/new/photos/services/user-entity"; import { EnteFile } from "@/new/photos/types/file"; import * as chrono from "chrono-node"; import { t } from "i18next"; @@ -25,7 +27,7 @@ import { import ComlinkSearchWorker from "utils/comlink/ComlinkSearchWorker"; import { getUniqueFiles } from "utils/file"; import { getFormattedDate } from "utils/search"; -import { getLatestEntities } from "./entityService"; +import { getEntityKey, getLatestEntities } from "./entityService"; import locationSearchService, { City } from "./locationSearchService"; const DIGITS = new Set(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]); @@ -415,6 +417,15 @@ function convertSuggestionToSearchQuery(option: Suggestion): Search { } async function getAllPeople(limit: number = undefined) { + if (!(await wipClusterEnable())) return []; + + const entityKey = await getEntityKey("person_v2" as EntityType); + const peopleR = await personDiff(entityKey.data); + const r = peopleR.length; + log.debug(() => ["people", peopleR]); + + if (r) return []; + let people: Array = []; // await mlIDbStorage.getAllPeople(); people = await wipCluster(); // await mlPeopleStore.iterate((person) => { @@ -425,5 +436,6 @@ async function getAllPeople(limit: number = undefined) { .sort((p1, p2) => p2.files.length - p1.files.length) .slice(0, limit); // log.debug(() => ["getAllPeople", result]); + return result; } diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index bee9291563..7f44fa2010 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -314,9 +314,15 @@ let last: Person[] | undefined; /** * WIP! Don't enable, dragon eggs are hatching here. */ +export const wipClusterEnable = async () => { + if (!isDevBuild || !(await isInternalUser())) return false; + if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return false; + if (last) return false; + return true; +}; + export const wipCluster = async () => { - if (!isDevBuild || !(await isInternalUser())) return; - if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return; + if (!(await wipClusterEnable())) return; if (last) return last; diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 939ed9d4e9..e7b08b822e 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -2,6 +2,7 @@ import { decryptMetadataBytes } from "@/base/crypto/ente"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; import { z } from "zod"; +import { gunzip } from "./gzip"; /** * User entities are predefined lists of otherwise arbitrary data that the user @@ -117,3 +118,17 @@ export const userEntityDiff = async ( ), ); }; + +/** + * Fetch all Person entities that have been created or updated since the last + * time we checked. + */ +export const personDiff = async (entityKeyB64: string) => { + const entities = await userEntityDiff("person_v2", 0, entityKeyB64); + return Promise.all( + entities.map(async ({ data }) => { + if (!data) return undefined; + return JSON.parse(await gunzip(data)) as unknown; + }), + ); +}; From cd2fde2c2e9e38f32e6c6407a27ad76a2f8cfa2a Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 9 Aug 2024 15:50:29 +0530 Subject: [PATCH 124/211] [server] 1/n Support for persisting preview video --- server/cmd/museum/main.go | 9 +- server/ente/filedata/filedata.go | 100 ++++++++++++------ server/ente/filedata/putfiledata.go | 24 +++-- server/pkg/api/file.go | 80 +++++++------- server/pkg/api/file_data.go | 52 +++++---- server/pkg/controller/file_preview.go | 68 ------------ server/pkg/controller/filedata/controller.go | 33 +++++- .../pkg/controller/filedata/preview_files.go | 54 ++++++++++ server/pkg/controller/filedata/s3.go | 40 +++++++ 9 files changed, 276 insertions(+), 184 deletions(-) create mode 100644 server/pkg/controller/filedata/preview_files.go diff --git a/server/cmd/museum/main.go b/server/cmd/museum/main.go index 7aa2f41c1f..a34276bad4 100644 --- a/server/cmd/museum/main.go +++ b/server/cmd/museum/main.go @@ -413,12 +413,11 @@ func main() { privateAPI.GET("/files/preview/:fileID", fileHandler.GetThumbnail) privateAPI.GET("/files/preview/v2/:fileID", fileHandler.GetThumbnail) - privateAPI.GET("/files/file-data/playlist/:fileID", fileHandler.GetVideoPlaylist) - privateAPI.POST("/files/file-data/playlist", fileHandler.ReportVideoPlayList) - privateAPI.GET("/files/file-data/preview/upload-url/:fileID", fileHandler.GetVideoUploadURL) - privateAPI.GET("/files/file-data/preview/:fileID", fileHandler.GetVideoPreviewUrl) privateAPI.PUT("/files/data/", fileHandler.PutFileData) - privateAPI.POST("files/fetch-data/", fileHandler.GetFilesData) + privateAPI.POST("files/data/fetch", fileHandler.GetFilesData) + privateAPI.GET("files/data/fetch", fileHandler.GetFileData) + privateAPI.GET("/files/data/preview-upload-url/", fileHandler.GetPreviewUploadURL) + privateAPI.GET("/files/data/preview/", fileHandler.GetPreviewURL) privateAPI.POST("/files", fileHandler.CreateOrUpdate) privateAPI.POST("/files/copy", fileHandler.CopyFiles) diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go index f9cfccd28e..9d5f366351 100644 --- a/server/ente/filedata/filedata.go +++ b/server/ente/filedata/filedata.go @@ -5,9 +5,16 @@ import ( "github.com/ente-io/museum/ente" ) +type Entity struct { + FileID int64 `json:"fileID"` + Type ente.ObjectType `json:"type"` + EncryptedData string `json:"encryptedData"` + DecryptionHeader string `json:"decryptionHeader"` +} + // GetFilesData should only be used for getting the preview video playlist and derived metadata. type GetFilesData struct { - FileIDs []int64 `form:"fileIDs" binding:"required"` + FileIDs []int64 `json:"fileIDs" binding:"required"` Type ente.ObjectType `json:"type" binding:"required"` } @@ -24,38 +31,16 @@ func (g *GetFilesData) Validate() error { return nil } -type Entity struct { - FileID int64 `json:"fileID"` - Type ente.ObjectType `json:"type"` - EncryptedData string `json:"encryptedData"` - DecryptionHeader string `json:"decryptionHeader"` -} - -// Row represents the data that is stored in the file_data table. -type Row struct { - FileID int64 - UserID int64 - Type ente.ObjectType - Size int64 - LatestBucket string - ReplicatedBuckets []string - DeleteFromBuckets []string - InflightReplicas []string - PendingSync bool - IsDeleted bool - SyncLockedTill int64 - CreatedAt int64 - UpdatedAt int64 +type GetFileData struct { + FileID int64 `form:"fileID" binding:"required"` + Type ente.ObjectType `form:"type" binding:"required"` } -func (r Row) S3FileMetadataObjectKey() string { - if r.Type == ente.DerivedMeta { - return derivedMetaPath(r.FileID, r.UserID) - } - if r.Type == ente.PreviewVideo { - return previewVideoPlaylist(r.FileID, r.UserID) +func (g *GetFileData) Validate() error { + if g.Type != ente.PreviewVideo && g.Type != ente.DerivedMeta { + return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) } - panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) + return nil } type GetFilesDataResponse struct { @@ -73,14 +58,65 @@ type S3FileMetadata struct { Client string `json:"client"` } -type GetPreviewUrlRequest struct { +type GetPreviewURLRequest struct { FileID int64 `form:"fileID" binding:"required"` Type ente.ObjectType `form:"type" binding:"required"` } -func (g *GetPreviewUrlRequest) Validate() error { +func (g *GetPreviewURLRequest) Validate() error { if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage { return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) } return nil } + +type PreviewUploadUrlRequest struct { + FileID int64 `form:"fileID" binding:"required"` + Type ente.ObjectType `form:"type" binding:"required"` +} + +func (g *PreviewUploadUrlRequest) Validate() error { + if g.Type != ente.PreviewVideo && g.Type != ente.PreviewImage { + return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) + } + return nil +} + +// Row represents the data that is stored in the file_data table. +type Row struct { + FileID int64 + UserID int64 + Type ente.ObjectType + // If a file type has multiple objects, then the size is the sum of all the objects. + Size int64 + LatestBucket string + ReplicatedBuckets []string + DeleteFromBuckets []string + InflightReplicas []string + PendingSync bool + IsDeleted bool + SyncLockedTill int64 + CreatedAt int64 + UpdatedAt int64 +} + +// S3FileMetadataObjectKey returns the object key for the metadata stored in the S3 bucket. +func (r *Row) S3FileMetadataObjectKey() string { + if r.Type == ente.DerivedMeta { + return derivedMetaPath(r.FileID, r.UserID) + } + if r.Type == ente.PreviewVideo { + return previewVideoPlaylist(r.FileID, r.UserID) + } + panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) +} + +// GetS3FileObjectKey returns the object key for the file data stored in the S3 bucket. +func (r *Row) GetS3FileObjectKey() string { + if r.Type == ente.PreviewVideo { + return previewVideoPath(r.FileID, r.UserID) + } else if r.Type == ente.PreviewImage { + return previewImagePath(r.FileID, r.UserID) + } + panic(fmt.Sprintf("unsupported object type %s", r.Type)) +} diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go index dcdab16c19..0d70570da3 100644 --- a/server/ente/filedata/putfiledata.go +++ b/server/ente/filedata/putfiledata.go @@ -17,23 +17,27 @@ type PutFileDataRequest struct { Version *int `json:"version,omitempty"` } +func (r PutFileDataRequest) isEncDataPresent() bool { + return r.EncryptedData != nil && r.DecryptionHeader != nil && *r.EncryptedData != "" && *r.DecryptionHeader != "" +} + +func (r PutFileDataRequest) isObjectDataPresent() bool { + return r.ObjectKey != nil && *r.ObjectKey != "" && r.ObjectSize != nil && *r.ObjectSize > 0 +} + func (r PutFileDataRequest) Validate() error { switch r.Type { case ente.PreviewVideo: - if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { - // the video playlist is uploaded as part of encrypted data and decryption header - return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for preview video") - } - if r.ObjectSize == nil || r.ObjectKey == nil { - return ente.NewBadRequestWithMessage("size and objectKey are required for preview video") + if !r.isEncDataPresent() || !r.isObjectDataPresent() { + return ente.NewBadRequestWithMessage("object and metadata are required") } case ente.PreviewImage: - if r.ObjectSize == nil || r.ObjectKey == nil { - return ente.NewBadRequestWithMessage("size and objectKey are required for preview image") + if !r.isObjectDataPresent() || r.isEncDataPresent() { + return ente.NewBadRequestWithMessage("object (only) data is required for preview image") } case ente.DerivedMeta: - if r.EncryptedData == nil || r.DecryptionHeader == nil || *r.EncryptedData == "" || *r.DecryptionHeader == "" { - return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader are required for derived meta") + if !r.isEncDataPresent() || r.isObjectDataPresent() { + return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader (only) are required for derived meta") } default: return ente.NewBadRequestWithMessage(fmt.Sprintf("invalid object type %s", r.Type)) diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go index 2e15ade325..65efb1d068 100644 --- a/server/pkg/api/file.go +++ b/server/pkg/api/file.go @@ -33,7 +33,7 @@ const DefaultMaxBatchSize = 1000 const DefaultCopyBatchSize = 100 // CreateOrUpdate creates an entry for a file -func (h *FileHandler) CreateOrUpdate(c *gin.Context) { +func (f *FileHandler) CreateOrUpdate(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) var file ente.File if err := c.ShouldBindJSON(&file); err != nil { @@ -48,7 +48,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) { if file.ID == 0 { file.OwnerID = userID file.IsDeleted = false - file, err := h.Controller.Create(c, userID, file, c.Request.UserAgent(), enteApp) + file, err := f.Controller.Create(c, userID, file, c.Request.UserAgent(), enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -56,7 +56,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) { c.JSON(http.StatusOK, file) return } - response, err := h.Controller.Update(c, userID, file, enteApp) + response, err := f.Controller.Update(c, userID, file, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -65,7 +65,7 @@ func (h *FileHandler) CreateOrUpdate(c *gin.Context) { } // CopyFiles copies files that are owned by another user -func (h *FileHandler) CopyFiles(c *gin.Context) { +func (f *FileHandler) CopyFiles(c *gin.Context) { var req ente.CopyFileSyncRequest if err := c.ShouldBindJSON(&req); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -75,7 +75,7 @@ func (h *FileHandler) CopyFiles(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.NewBadRequestWithMessage(fmt.Sprintf("more than %d items", DefaultCopyBatchSize)), "")) return } - response, err := h.FileCopyCtrl.CopyFiles(c, req) + response, err := f.FileCopyCtrl.CopyFiles(c, req) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -84,7 +84,7 @@ func (h *FileHandler) CopyFiles(c *gin.Context) { } // Update updates already existing file -func (h *FileHandler) Update(c *gin.Context) { +func (f *FileHandler) Update(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) @@ -98,7 +98,7 @@ func (h *FileHandler) Update(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, "fileID should be >0")) return } - response, err := h.Controller.Update(c, userID, file, enteApp) + response, err := f.Controller.Update(c, userID, file, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -107,12 +107,12 @@ func (h *FileHandler) Update(c *gin.Context) { } // GetUploadURLs returns a bunch of urls where in the user can upload objects -func (h *FileHandler) GetUploadURLs(c *gin.Context) { +func (f *FileHandler) GetUploadURLs(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) count, _ := strconv.Atoi(c.Query("count")) - urls, err := h.Controller.GetUploadURLs(c, userID, count, enteApp, false) + urls, err := f.Controller.GetUploadURLs(c, userID, count, enteApp, false) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -123,12 +123,12 @@ func (h *FileHandler) GetUploadURLs(c *gin.Context) { } // GetMultipartUploadURLs returns an array of PartUpload PresignedURLs -func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) { +func (f *FileHandler) GetMultipartUploadURLs(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) count, _ := strconv.Atoi(c.Query("count")) - urls, err := h.Controller.GetMultipartUploadURLs(c, userID, count, enteApp) + urls, err := f.Controller.GetMultipartUploadURLs(c, userID, count, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -139,21 +139,21 @@ func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) { } // Get redirects the request to the file location -func (h *FileHandler) Get(c *gin.Context) { +func (f *FileHandler) Get(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetFileURL(c, userID, fileID) + url, err := f.Controller.GetFileURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - h.logBadRedirect(c) + f.logBadRedirect(c) c.Redirect(http.StatusTemporaryRedirect, url) } // GetV2 returns the URL of the file to client -func (h *FileHandler) GetV2(c *gin.Context) { +func (f *FileHandler) GetV2(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetFileURL(c, userID, fileID) + url, err := f.Controller.GetFileURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -164,21 +164,21 @@ func (h *FileHandler) GetV2(c *gin.Context) { } // GetThumbnail redirects the request to the file's thumbnail location -func (h *FileHandler) GetThumbnail(c *gin.Context) { +func (f *FileHandler) GetThumbnail(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetThumbnailURL(c, userID, fileID) + url, err := f.Controller.GetThumbnailURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - h.logBadRedirect(c) + f.logBadRedirect(c) c.Redirect(http.StatusTemporaryRedirect, url) } // GetThumbnailV2 returns the URL of the thumbnail to the client -func (h *FileHandler) GetThumbnailV2(c *gin.Context) { +func (f *FileHandler) GetThumbnailV2(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetThumbnailURL(c, userID, fileID) + url, err := f.Controller.GetThumbnailURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -189,7 +189,7 @@ func (h *FileHandler) GetThumbnailV2(c *gin.Context) { } // Trash moves the given files to the trash bin -func (h *FileHandler) Trash(c *gin.Context) { +func (f *FileHandler) Trash(c *gin.Context) { var request ente.TrashRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "failed to bind")) @@ -201,7 +201,7 @@ func (h *FileHandler) Trash(c *gin.Context) { } userID := auth.GetUserID(c.Request.Header) request.OwnerID = userID - err := h.Controller.Trash(c, userID, request) + err := f.Controller.Trash(c, userID, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -210,7 +210,7 @@ func (h *FileHandler) Trash(c *gin.Context) { } // GetSize returns the size of files indicated by fileIDs -func (h *FileHandler) GetSize(c *gin.Context) { +func (f *FileHandler) GetSize(c *gin.Context) { var request ente.FileIDsRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -227,7 +227,7 @@ func (h *FileHandler) GetSize(c *gin.Context) { return } - size, err := h.Controller.GetSize(userID, request.FileIDs) + size, err := f.Controller.GetSize(userID, request.FileIDs) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -238,7 +238,7 @@ func (h *FileHandler) GetSize(c *gin.Context) { } // GetInfo returns the FileInfo of files indicated by fileIDs -func (h *FileHandler) GetInfo(c *gin.Context) { +func (f *FileHandler) GetInfo(c *gin.Context) { var request ente.FileIDsRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "failed to bind request")) @@ -246,7 +246,7 @@ func (h *FileHandler) GetInfo(c *gin.Context) { } userID := auth.GetUserID(c.Request.Header) - response, err := h.Controller.GetFileInfo(c, userID, request.FileIDs) + response, err := f.Controller.GetFileInfo(c, userID, request.FileIDs) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -295,9 +295,9 @@ func shouldRejectRequest(c *gin.Context) (bool, error) { } // GetDuplicates returns the list of files of the same size -func (h *FileHandler) GetDuplicates(c *gin.Context) { +func (f *FileHandler) GetDuplicates(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) - dupes, err := h.Controller.GetDuplicates(userID) + dupes, err := f.Controller.GetDuplicates(userID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -308,10 +308,10 @@ func (h *FileHandler) GetDuplicates(c *gin.Context) { } // GetLargeThumbnail returns the list of files whose thumbnail size is larger than threshold size -func (h *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { +func (f *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) threshold, _ := strconv.ParseInt(c.Query("threshold"), 10, 64) - largeThumbnailFiles, err := h.Controller.GetLargeThumbnailFiles(userID, threshold) + largeThumbnailFiles, err := f.Controller.GetLargeThumbnailFiles(userID, threshold) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -322,7 +322,7 @@ func (h *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { } // UpdateMagicMetadata updates magic metadata for a list of files. -func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) { +func (f *FileHandler) UpdateMagicMetadata(c *gin.Context) { var request ente.UpdateMultipleMagicMetadataRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -332,7 +332,7 @@ func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.ErrBatchSizeTooLarge, "")) return } - err := h.Controller.UpdateMagicMetadata(c, request, false) + err := f.Controller.UpdateMagicMetadata(c, request, false) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -341,13 +341,13 @@ func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) { } // UpdatePublicMagicMetadata updates public magic metadata for a list of files. -func (h *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { +func (f *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { var request ente.UpdateMultipleMagicMetadataRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - err := h.Controller.UpdateMagicMetadata(c, request, true) + err := f.Controller.UpdateMagicMetadata(c, request, true) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -356,7 +356,7 @@ func (h *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { } // UpdateThumbnail updates thumbnail of a file -func (h *FileHandler) UpdateThumbnail(c *gin.Context) { +func (f *FileHandler) UpdateThumbnail(c *gin.Context) { enteApp := auth.GetApp(c) var request ente.UpdateThumbnailRequest @@ -364,7 +364,7 @@ func (h *FileHandler) UpdateThumbnail(c *gin.Context) { handler.Error(c, stacktrace.Propagate(err, "")) return } - err := h.Controller.UpdateThumbnail(c, request.FileID, request.Thumbnail, enteApp) + err := f.Controller.UpdateThumbnail(c, request.FileID, request.Thumbnail, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -372,8 +372,8 @@ func (h *FileHandler) UpdateThumbnail(c *gin.Context) { c.Status(http.StatusOK) } -func (h *FileHandler) GetTotalFileCount(c *gin.Context) { - count, err := h.Controller.GetTotalFileCount() +func (f *FileHandler) GetTotalFileCount(c *gin.Context) { + count, err := f.Controller.GetTotalFileCount() if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -390,7 +390,7 @@ func getUserAndFileIDs(c *gin.Context) (int64, int64) { } // logBadRedirect will log the request id if we are redirecting to another url with the auth-token in header -func (h *FileHandler) logBadRedirect(c *gin.Context) { +func (f *FileHandler) logBadRedirect(c *gin.Context) { if len(c.GetHeader("X-Auth-Token")) != 0 && os.Getenv("ENVIRONMENT") != "" { log.WithField("req_id", requestid.Get(c)).Error("critical: sending token to another service") } diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go index 50015c3d4d..d45a3392b8 100644 --- a/server/pkg/api/file_data.go +++ b/server/pkg/api/file_data.go @@ -4,12 +4,10 @@ import ( "fmt" "github.com/ente-io/museum/ente" fileData "github.com/ente-io/museum/ente/filedata" - "github.com/ente-io/museum/pkg/utils/auth" "github.com/ente-io/museum/pkg/utils/handler" "github.com/ente-io/stacktrace" "github.com/gin-gonic/gin" "net/http" - "strconv" ) func (f *FileHandler) PutFileData(ctx *gin.Context) { @@ -50,50 +48,50 @@ func (f *FileHandler) GetFilesData(ctx *gin.Context) { ctx.JSON(http.StatusOK, resp) } -func (h *FileHandler) GetVideoUploadURL(c *gin.Context) { - enteApp := auth.GetApp(c) - userID, fileID := getUserAndFileIDs(c) - urls, err := h.Controller.GetVideoUploadUrl(c, userID, fileID, enteApp) - if err != nil { - handler.Error(c, stacktrace.Propagate(err, "")) +func (f *FileHandler) GetFileData(ctx *gin.Context) { + var req fileData.GetFileData + if err := ctx.ShouldBindJSON(&req); err != nil { + ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error())) return } - c.JSON(http.StatusOK, urls) -} - -func (h *FileHandler) GetVideoPreviewUrl(c *gin.Context) { - userID, fileID := getUserAndFileIDs(c) - url, err := h.Controller.GetPreviewUrl(c, userID, fileID) + resp, err := f.FileDataCtrl.GetFileData(ctx, req) if err != nil { - handler.Error(c, stacktrace.Propagate(err, "")) + handler.Error(ctx, err) return } - c.JSON(http.StatusOK, gin.H{ - "url": url, + ctx.JSON(http.StatusOK, gin.H{ + "data": resp, }) } -func (h *FileHandler) ReportVideoPlayList(c *gin.Context) { - var request ente.InsertOrUpdateEmbeddingRequest +func (f *FileHandler) GetPreviewUploadURL(c *gin.Context) { + var request fileData.PreviewUploadUrlRequest if err := c.ShouldBindJSON(&request); err != nil { - handler.Error(c, - stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) + handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) return } - err := h.Controller.ReportVideoPreview(c, request) + url, err := f.FileDataCtrl.PreviewUploadURL(c, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - c.Status(http.StatusOK) + c.JSON(http.StatusOK, gin.H{ + "url": url, + }) } -func (h *FileHandler) GetVideoPlaylist(c *gin.Context) { - fileID, _ := strconv.ParseInt(c.Param("fileID"), 10, 64) - response, err := h.Controller.GetPlaylist(c, fileID) +func (f *FileHandler) GetPreviewURL(c *gin.Context) { + var request fileData.GetPreviewURLRequest + if err := c.ShouldBindJSON(&request); err != nil { + handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) + return + } + url, err := f.FileDataCtrl.GetPreviewUrl(c, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - c.JSON(http.StatusOK, response) + c.JSON(http.StatusOK, gin.H{ + "url": url, + }) } diff --git a/server/pkg/controller/file_preview.go b/server/pkg/controller/file_preview.go index d998267319..5996bc4f70 100644 --- a/server/pkg/controller/file_preview.go +++ b/server/pkg/controller/file_preview.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" "github.com/ente-io/museum/ente" @@ -22,53 +21,6 @@ const ( _model = "hls_video" ) -// GetUploadURLs returns a bunch of presigned URLs for uploading files -func (c *FileController) GetVideoUploadUrl(ctx context.Context, userID int64, fileID int64, app ente.App) (*ente.UploadURL, error) { - err := c.UsageCtrl.CanUploadFile(ctx, userID, nil, app) - if err != nil { - return nil, stacktrace.Propagate(err, "") - } - s3Client := c.S3Config.GetDerivedStorageS3Client() - dc := c.S3Config.GetDerivedStorageDataCenter() - bucket := c.S3Config.GetDerivedStorageBucket() - objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/" + _model - url, err := c.getObjectURL(s3Client, dc, bucket, objectKey) - if err != nil { - return nil, stacktrace.Propagate(err, "") - } - log.Infof("Got upload URL for %s", objectKey) - return &url, nil -} - -func (c *FileController) GetPreviewUrl(ctx context.Context, userID int64, fileID int64) (string, error) { - err := c.verifyFileAccess(userID, fileID) - if err != nil { - return "", err - } - objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video" - // check if playlist exists - err = c.checkObjectExists(ctx, objectKey+"_playlist.m3u8", c.S3Config.GetDerivedStorageDataCenter()) - if err != nil { - return "", stacktrace.Propagate(ente.NewBadRequestWithMessage("Video playlist does not exist"), fmt.Sprintf("objectKey: %s", objectKey)) - } - s3Client := c.S3Config.GetDerivedStorageS3Client() - r, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{ - Bucket: c.S3Config.GetDerivedStorageBucket(), - Key: &objectKey, - }) - return r.Presign(PreSignedRequestValidityDuration) -} - -func (c *FileController) GetPlaylist(ctx *gin.Context, fileID int64) (ente.EmbeddingObject, error) { - objectKey := strconv.FormatInt(auth.GetUserID(ctx.Request.Header), 10) + "/ml-data/" + strconv.FormatInt(fileID, 10) + "/hls_video_playlist.m3u8" - // check if object exists - err := c.checkObjectExists(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter()) - if err != nil { - return ente.EmbeddingObject{}, stacktrace.Propagate(ente.NewBadRequestWithMessage("Video playlist does not exist"), fmt.Sprintf("objectKey: %s", objectKey)) - } - return c.downloadObject(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter()) -} - func (c *FileController) ReportVideoPreview(ctx *gin.Context, req ente.InsertOrUpdateEmbeddingRequest) error { userID := auth.GetUserID(ctx.Request.Header) if strings.Compare(req.Model, "hls_video") != 0 { @@ -128,26 +80,6 @@ func (c *FileController) uploadObject(obj ente.EmbeddingObject, key string, dc s return len(embeddingObj), nil } -func (c *FileController) downloadObject(ctx context.Context, objectKey string, dc string) (ente.EmbeddingObject, error) { - var obj ente.EmbeddingObject - buff := &aws.WriteAtBuffer{} - bucket := c.S3Config.GetBucket(dc) - s3Client := c.S3Config.GetS3Client(dc) - downloader := s3manager.NewDownloaderWithClient(&s3Client) - _, err := downloader.DownloadWithContext(ctx, buff, &s3.GetObjectInput{ - Bucket: bucket, - Key: &objectKey, - }) - if err != nil { - return obj, err - } - err = json.Unmarshal(buff.Bytes(), &obj) - if err != nil { - return obj, stacktrace.Propagate(err, "unmarshal failed") - } - return obj, nil -} - func (c *FileController) checkObjectExists(ctx context.Context, objectKey string, dc string) error { s3Client := c.S3Config.GetS3Client(dc) _, err := s3Client.HeadObject(&s3.HeadObjectInput{ diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index afd0a73a6f..b01cec9c48 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -83,10 +83,13 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR return stacktrace.Propagate(err, "validation failed") } userID := auth.GetUserID(ctx.Request.Header) - err := c._validateInsertPermission(ctx, req.FileID, userID) + err := c._validatePermission(ctx, req.FileID, userID) if err != nil { return stacktrace.Propagate(err, "") } + if req.Type != ente.DerivedMeta { + return stacktrace.Propagate(ente.NewBadRequestWithMessage("unsupported object type "+string(req.Type)), "") + } fileOwnerID := userID objectKey := req.S3FileMetadataObjectKey(fileOwnerID) obj := fileData.S3FileMetadata{ @@ -115,6 +118,32 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR return nil } +func (c *Controller) GetFileData(ctx *gin.Context, req fileData.GetFileData) (*fileData.Entity, error) { + if err := req.Validate(); err != nil { + return nil, stacktrace.Propagate(err, "validation failed") + } + if err := c._validatePermission(ctx, req.FileID, auth.GetUserID(ctx.Request.Header)); err != nil { + return nil, stacktrace.Propagate(err, "") + } + doRows, err := c.Repo.GetFilesData(ctx, req.Type, []int64{req.FileID}) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + if len(doRows) == 0 || doRows[0].IsDeleted { + return nil, stacktrace.Propagate(ente.ErrNotFound, "") + } + s3MetaObject, err := c.fetchS3FileMetadata(context.Background(), doRows[0], doRows[0].LatestBucket) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &fileData.Entity{ + FileID: doRows[0].FileID, + Type: doRows[0].Type, + EncryptedData: s3MetaObject.EncryptedData, + DecryptionHeader: s3MetaObject.DecryptionHeader, + }, nil +} + func (c *Controller) GetFilesData(ctx *gin.Context, req fileData.GetFilesData) (*fileData.GetFilesDataResponse, error) { userID := auth.GetUserID(ctx.Request.Header) if err := c._validateGetFilesData(ctx, userID, req); err != nil { @@ -254,7 +283,7 @@ func (c *Controller) _validateGetFilesData(ctx *gin.Context, userID int64, req f return nil } -func (c *Controller) _validateInsertPermission(ctx *gin.Context, fileID int64, actorID int64) error { +func (c *Controller) _validatePermission(ctx *gin.Context, fileID int64, actorID int64) error { err := c.AccessCtrl.VerifyFileOwnership(ctx, &access.VerifyFileOwnershipParams{ ActorUserId: actorID, FileIDs: []int64{fileID}, diff --git a/server/pkg/controller/filedata/preview_files.go b/server/pkg/controller/filedata/preview_files.go new file mode 100644 index 0000000000..4d6b0113ba --- /dev/null +++ b/server/pkg/controller/filedata/preview_files.go @@ -0,0 +1,54 @@ +package filedata + +import ( + "fmt" + "github.com/ente-io/museum/ente" + "github.com/ente-io/museum/ente/filedata" + "github.com/ente-io/museum/pkg/utils/auth" + "github.com/ente-io/stacktrace" + "github.com/gin-gonic/gin" +) + +func (c *Controller) GetPreviewUrl(ctx *gin.Context, request filedata.GetPreviewURLRequest) (*string, error) { + if err := request.Validate(); err != nil { + return nil, err + } + actorUser := auth.GetUserID(ctx.Request.Header) + if err := c._validatePermission(ctx, request.FileID, actorUser); err != nil { + return nil, err + } + data, err := c.Repo.GetFilesData(ctx, request.Type, []int64{request.FileID}) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + if len(data) == 0 || data[0].IsDeleted { + return nil, stacktrace.Propagate(ente.ErrNotFound, "") + } + enteUrl, err := c.signedUrlGet(data[0].LatestBucket, data[0].GetS3FileObjectKey()) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &enteUrl.URL, nil +} + +func (c *Controller) PreviewUploadURL(ctx *gin.Context, request filedata.PreviewUploadUrlRequest) (*string, error) { + if err := request.Validate(); err != nil { + return nil, err + } + actorUser := auth.GetUserID(ctx.Request.Header) + if err := c._validatePermission(ctx, request.FileID, actorUser); err != nil { + return nil, err + } + fileOwnerID, err := c.FileRepo.GetOwnerID(request.FileID) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + // note: instead of the final url, give a temp url for upload purpose. + uploadUrl := fmt.Sprintf("%s_temp_upload", filedata.PreviewUrl(request.FileID, fileOwnerID, request.Type)) + bucketID := c.S3Config.GetBucketID(request.Type) + enteUrl, err := c.getUploadURL(bucketID, uploadUrl) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &enteUrl.URL, nil +} diff --git a/server/pkg/controller/filedata/s3.go b/server/pkg/controller/filedata/s3.go index f8a55052ec..415e28cb28 100644 --- a/server/pkg/controller/filedata/s3.go +++ b/server/pkg/controller/filedata/s3.go @@ -7,11 +7,51 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" + "github.com/ente-io/museum/ente" fileData "github.com/ente-io/museum/ente/filedata" "github.com/ente-io/stacktrace" log "github.com/sirupsen/logrus" + stime "time" ) +const PreSignedRequestValidityDuration = 7 * 24 * stime.Hour + +func (c *Controller) getUploadURL(dc string, objectKey string) (*ente.UploadURL, error) { + s3Client := c.S3Config.GetS3Client(dc) + r, _ := s3Client.PutObjectRequest(&s3.PutObjectInput{ + Bucket: c.S3Config.GetBucket(dc), + Key: &objectKey, + }) + url, err := r.Presign(PreSignedRequestValidityDuration) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + err = c.ObjectCleanupController.AddTempObjectKey(objectKey, dc) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &ente.UploadURL{ + ObjectKey: objectKey, + URL: url, + }, nil +} + +func (c *Controller) signedUrlGet(dc string, objectKey string) (*ente.UploadURL, error) { + s3Client := c.S3Config.GetS3Client(dc) + r, _ := s3Client.GetObjectRequest(&s3.GetObjectInput{ + Bucket: c.S3Config.GetBucket(dc), + Key: &objectKey, + }) + url, err := r.Presign(PreSignedRequestValidityDuration) + if err != nil { + return nil, stacktrace.Propagate(err, "") + } + return &ente.UploadURL{ObjectKey: objectKey, URL: url}, nil +} + func (c *Controller) downloadObject(ctx context.Context, objectKey string, dc string) (fileData.S3FileMetadata, error) { var obj fileData.S3FileMetadata buff := &aws.WriteAtBuffer{} From 251a62721948208c55a5f0d0c0be2bea6519520a Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Fri, 9 Aug 2024 15:51:38 +0530 Subject: [PATCH 125/211] delete --- server/pkg/controller/file_preview.go | 93 --------------------------- 1 file changed, 93 deletions(-) delete mode 100644 server/pkg/controller/file_preview.go diff --git a/server/pkg/controller/file_preview.go b/server/pkg/controller/file_preview.go deleted file mode 100644 index 5996bc4f70..0000000000 --- a/server/pkg/controller/file_preview.go +++ /dev/null @@ -1,93 +0,0 @@ -package controller - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - "github.com/ente-io/museum/ente" - "github.com/ente-io/museum/pkg/utils/auth" - "github.com/ente-io/museum/pkg/utils/network" - "github.com/ente-io/stacktrace" - "github.com/gin-gonic/gin" - log "github.com/sirupsen/logrus" - "strconv" - "strings" -) - -const ( - _model = "hls_video" -) - -func (c *FileController) ReportVideoPreview(ctx *gin.Context, req ente.InsertOrUpdateEmbeddingRequest) error { - userID := auth.GetUserID(ctx.Request.Header) - if strings.Compare(req.Model, "hls_video") != 0 { - return stacktrace.Propagate(ente.NewBadRequestWithMessage("Model should be hls_video"), "Invalid fileID") - } - count, err := c.CollectionRepo.GetCollectionCount(req.FileID) - if err != nil { - return stacktrace.Propagate(err, "") - } - if count < 1 { - return stacktrace.Propagate(ente.ErrNotFound, "") - } - version := 1 - if req.Version != nil { - version = *req.Version - } - objectKey := strconv.FormatInt(userID, 10) + "/ml-data/" + strconv.FormatInt(req.FileID, 10) + "/hls_video" - playlistKey := objectKey + "_playlist.m3u8" - - // verify that objectKey exists - err = c.checkObjectExists(ctx, objectKey, c.S3Config.GetDerivedStorageDataCenter()) - if err != nil { - return stacktrace.Propagate(ente.NewBadRequestWithMessage("Video object does not exist, upload that before playlist reporting"), fmt.Sprintf("objectKey: %s", objectKey)) - } - - obj := ente.EmbeddingObject{ - Version: version, - EncryptedEmbedding: req.EncryptedEmbedding, - DecryptionHeader: req.DecryptionHeader, - Client: network.GetClientInfo(ctx), - } - _, uploadErr := c.uploadObject(obj, playlistKey, c.S3Config.GetDerivedStorageDataCenter()) - if uploadErr != nil { - log.Error(uploadErr) - return stacktrace.Propagate(uploadErr, "") - } - return nil -} - -func (c *FileController) uploadObject(obj ente.EmbeddingObject, key string, dc string) (int, error) { - embeddingObj, _ := json.Marshal(obj) - s3Client := c.S3Config.GetS3Client(dc) - s3Bucket := c.S3Config.GetBucket(dc) - uploader := s3manager.NewUploaderWithClient(&s3Client) - up := s3manager.UploadInput{ - Bucket: s3Bucket, - Key: &key, - Body: bytes.NewReader(embeddingObj), - } - result, err := uploader.Upload(&up) - if err != nil { - log.Error(err) - return -1, stacktrace.Propagate(err, "") - } - - log.Infof("Uploaded to bucket %s", result.Location) - return len(embeddingObj), nil -} - -func (c *FileController) checkObjectExists(ctx context.Context, objectKey string, dc string) error { - s3Client := c.S3Config.GetS3Client(dc) - _, err := s3Client.HeadObject(&s3.HeadObjectInput{ - Bucket: c.S3Config.GetBucket(dc), - Key: &objectKey, - }) - if err != nil { - return err - } - return nil -} From 839727393b8fedb39d0574f6463bddab31b52a47 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 16:17:09 +0530 Subject: [PATCH 126/211] Debugging code --- web/apps/photos/src/services/searchService.ts | 12 +++++++----- web/packages/new/photos/services/user-entity.ts | 9 +++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 2bb7d304cb..97721a13ad 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -419,12 +419,14 @@ function convertSuggestionToSearchQuery(option: Suggestion): Search { async function getAllPeople(limit: number = undefined) { if (!(await wipClusterEnable())) return []; - const entityKey = await getEntityKey("person_v2" as EntityType); - const peopleR = await personDiff(entityKey.data); - const r = peopleR.length; - log.debug(() => ["people", peopleR]); + if (Math.random() < 0.01) { + const entityKey = await getEntityKey("person" as EntityType); + const peopleR = await personDiff(entityKey.data); + const r = peopleR.length; + log.debug(() => ["people", peopleR]); - if (r) return []; + if (r) return []; + } let people: Array = []; // await mlIDbStorage.getAllPeople(); people = await wipCluster(); diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index e7b08b822e..c6cca31cc6 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -2,7 +2,6 @@ import { decryptMetadataBytes } from "@/base/crypto/ente"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; import { z } from "zod"; -import { gunzip } from "./gzip"; /** * User entities are predefined lists of otherwise arbitrary data that the user @@ -11,11 +10,12 @@ import { gunzip } from "./gzip"; * e.g. location tags, people in their photos. */ export type EntityType = + | "person" /** * A new version of the Person entity where the data is gzipped before * encryption. */ - "person_v2"; + | "person_v2"; /** * The maximum number of items to fetch in a single diff @@ -124,11 +124,12 @@ export const userEntityDiff = async ( * time we checked. */ export const personDiff = async (entityKeyB64: string) => { - const entities = await userEntityDiff("person_v2", 0, entityKeyB64); + const entities = await userEntityDiff("person", 0, entityKeyB64); return Promise.all( entities.map(async ({ data }) => { if (!data) return undefined; - return JSON.parse(await gunzip(data)) as unknown; + // return JSON.parse(await gunzip(data)) as unknown; + return JSON.parse(new TextDecoder().decode(data)) as unknown; }), ); }; From 2d768c9c61588c21f8c4440493ddf989c06ad572 Mon Sep 17 00:00:00 2001 From: Aman Raj Singh Mourya Date: Fri, 9 Aug 2024 16:44:44 +0530 Subject: [PATCH 127/211] [mob][photos] Fallback for no device lock found for guest view and extracted strings --- mobile/lib/generated/intl/messages_cs.dart | 4 +-- mobile/lib/generated/intl/messages_de.dart | 4 +-- mobile/lib/generated/intl/messages_en.dart | 4 +-- mobile/lib/generated/intl/messages_es.dart | 4 +-- mobile/lib/generated/intl/messages_fr.dart | 4 +-- mobile/lib/generated/intl/messages_it.dart | 4 +-- mobile/lib/generated/intl/messages_ko.dart | 4 +-- mobile/lib/generated/intl/messages_nl.dart | 4 +-- mobile/lib/generated/intl/messages_no.dart | 4 +-- mobile/lib/generated/intl/messages_pl.dart | 4 +-- mobile/lib/generated/intl/messages_pt.dart | 4 +-- mobile/lib/generated/intl/messages_ru.dart | 4 +-- mobile/lib/generated/intl/messages_tr.dart | 4 +-- mobile/lib/generated/intl/messages_zh.dart | 4 +-- mobile/lib/generated/l10n.dart | 20 +++++------ mobile/lib/l10n/intl_cs.arb | 3 +- mobile/lib/l10n/intl_de.arb | 3 +- mobile/lib/l10n/intl_en.arb | 4 +-- mobile/lib/l10n/intl_es.arb | 3 +- mobile/lib/l10n/intl_fr.arb | 3 +- mobile/lib/l10n/intl_it.arb | 3 +- mobile/lib/l10n/intl_ko.arb | 3 +- mobile/lib/l10n/intl_nl.arb | 3 +- mobile/lib/l10n/intl_no.arb | 3 +- mobile/lib/l10n/intl_pl.arb | 3 +- mobile/lib/l10n/intl_pt.arb | 3 +- mobile/lib/l10n/intl_ru.arb | 3 +- mobile/lib/l10n/intl_tr.arb | 3 +- mobile/lib/l10n/intl_zh.arb | 3 +- .../file_selection_actions_widget.dart | 33 ++++++++++++------- mobile/lib/ui/viewer/file/file_app_bar.dart | 2 +- 31 files changed, 88 insertions(+), 66 deletions(-) diff --git a/mobile/lib/generated/intl/messages_cs.dart b/mobile/lib/generated/intl/messages_cs.dart index 6c8d749cc6..b49e60133c 100644 --- a/mobile/lib/generated/intl/messages_cs.dart +++ b/mobile/lib/generated/intl/messages_cs.dart @@ -61,6 +61,8 @@ class MessageLookup extends MessageLookupByLibrary { "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hideContent": MessageLookupByLibrary.simpleMessage("Hide content"), "hideContentDescriptionAndroid": MessageLookupByLibrary.simpleMessage( "Hides app content in the app switcher and disables screenshots"), @@ -105,8 +107,6 @@ class MessageLookup extends MessageLookupByLibrary { "setNewPassword": MessageLookupByLibrary.simpleMessage("Set new password"), "setNewPin": MessageLookupByLibrary.simpleMessage("Set new PIN"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "tapToUnlock": MessageLookupByLibrary.simpleMessage("Tap to unlock"), "thisWillRemovePublicLinksOfAllSelectedQuickLinks": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_de.dart b/mobile/lib/generated/intl/messages_de.dart index 6987ee0830..feed2c27a5 100644 --- a/mobile/lib/generated/intl/messages_de.dart +++ b/mobile/lib/generated/intl/messages_de.dart @@ -826,6 +826,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Fotos in der Nähe gruppieren"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Wir tracken keine App-Installationen. Es würde uns jedoch helfen, wenn du uns mitteilst, wie du von uns erfahren hast!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1514,8 +1516,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Verbesserung vorschlagen"), "support": MessageLookupByLibrary.simpleMessage("Support"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "Um die Sperre für die Wischfunktion zu aktivieren, richte bitte einen Gerätepasscode oder eine Bildschirmsperre in den Systemeinstellungen ein."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Synchronisierung angehalten"), diff --git a/mobile/lib/generated/intl/messages_en.dart b/mobile/lib/generated/intl/messages_en.dart index 6eafdbf79a..d22cdd7ecc 100644 --- a/mobile/lib/generated/intl/messages_en.dart +++ b/mobile/lib/generated/intl/messages_en.dart @@ -794,6 +794,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Group nearby photos"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "We don\'t track app installs. It\'d help if you told us where you found us!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1448,8 +1450,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Suggest features"), "support": MessageLookupByLibrary.simpleMessage("Support"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Sync stopped"), "syncing": MessageLookupByLibrary.simpleMessage("Syncing..."), diff --git a/mobile/lib/generated/intl/messages_es.dart b/mobile/lib/generated/intl/messages_es.dart index 25d102c0e6..857b10f70a 100644 --- a/mobile/lib/generated/intl/messages_es.dart +++ b/mobile/lib/generated/intl/messages_es.dart @@ -828,6 +828,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Agrupar fotos cercanas"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "No rastreamos las aplicaciones instaladas. ¡Nos ayudarías si nos dijeras dónde nos encontraste!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1522,8 +1524,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Sugerir una característica"), "support": MessageLookupByLibrary.simpleMessage("Soporte"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Sincronización detenida"), diff --git a/mobile/lib/generated/intl/messages_fr.dart b/mobile/lib/generated/intl/messages_fr.dart index 3abb699e48..cc0feb9555 100644 --- a/mobile/lib/generated/intl/messages_fr.dart +++ b/mobile/lib/generated/intl/messages_fr.dart @@ -776,6 +776,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Grouper les photos à proximité"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Nous ne suivons pas les installations d\'applications. Il serait utile que vous nous disiez comment vous nous avez trouvés !"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1404,8 +1406,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage( "Suggérer des fonctionnalités"), "support": MessageLookupByLibrary.simpleMessage("Support"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Synchronisation arrêtée ?"), diff --git a/mobile/lib/generated/intl/messages_it.dart b/mobile/lib/generated/intl/messages_it.dart index c978ee12d6..1681da9726 100644 --- a/mobile/lib/generated/intl/messages_it.dart +++ b/mobile/lib/generated/intl/messages_it.dart @@ -746,6 +746,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Raggruppa foto nelle vicinanze"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Non teniamo traccia del numero di installazioni dell\'app. Sarebbe utile se ci dicesse dove ci ha trovato!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1332,8 +1334,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Suggerisci una funzionalità"), "support": MessageLookupByLibrary.simpleMessage("Assistenza"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Sincronizzazione interrotta"), diff --git a/mobile/lib/generated/intl/messages_ko.dart b/mobile/lib/generated/intl/messages_ko.dart index 8b30ee0b86..5a372c298b 100644 --- a/mobile/lib/generated/intl/messages_ko.dart +++ b/mobile/lib/generated/intl/messages_ko.dart @@ -61,6 +61,8 @@ class MessageLookup extends MessageLookupByLibrary { "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hideContent": MessageLookupByLibrary.simpleMessage("Hide content"), "hideContentDescriptionAndroid": MessageLookupByLibrary.simpleMessage( "Hides app content in the app switcher and disables screenshots"), @@ -105,8 +107,6 @@ class MessageLookup extends MessageLookupByLibrary { "setNewPassword": MessageLookupByLibrary.simpleMessage("Set new password"), "setNewPin": MessageLookupByLibrary.simpleMessage("Set new PIN"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "tapToUnlock": MessageLookupByLibrary.simpleMessage("Tap to unlock"), "thisWillRemovePublicLinksOfAllSelectedQuickLinks": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_nl.dart b/mobile/lib/generated/intl/messages_nl.dart index fd84335925..b68312d437 100644 --- a/mobile/lib/generated/intl/messages_nl.dart +++ b/mobile/lib/generated/intl/messages_nl.dart @@ -829,6 +829,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Groep foto\'s in de buurt"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Wij gebruiken geen tracking. Het zou helpen als je ons vertelt waar je ons gevonden hebt!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1511,8 +1513,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Features voorstellen"), "support": MessageLookupByLibrary.simpleMessage("Ondersteuning"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "Om swipe-vergrendeling in te schakelen, stelt u de toegangscode van het apparaat of schermvergrendeling in uw systeeminstellingen in."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Synchronisatie gestopt"), diff --git a/mobile/lib/generated/intl/messages_no.dart b/mobile/lib/generated/intl/messages_no.dart index e737cd0591..886121b119 100644 --- a/mobile/lib/generated/intl/messages_no.dart +++ b/mobile/lib/generated/intl/messages_no.dart @@ -79,6 +79,8 @@ class MessageLookup extends MessageLookupByLibrary { "fileTypes": MessageLookupByLibrary.simpleMessage("File types"), "foundFaces": MessageLookupByLibrary.simpleMessage("Found faces"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hideContent": MessageLookupByLibrary.simpleMessage("Hide content"), "hideContentDescriptionAndroid": MessageLookupByLibrary.simpleMessage( "Hides app content in the app switcher and disables screenshots"), @@ -127,8 +129,6 @@ class MessageLookup extends MessageLookupByLibrary { "setNewPassword": MessageLookupByLibrary.simpleMessage("Set new password"), "setNewPin": MessageLookupByLibrary.simpleMessage("Set new PIN"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "tapToUnlock": MessageLookupByLibrary.simpleMessage("Tap to unlock"), "thisWillRemovePublicLinksOfAllSelectedQuickLinks": MessageLookupByLibrary.simpleMessage( diff --git a/mobile/lib/generated/intl/messages_pl.dart b/mobile/lib/generated/intl/messages_pl.dart index 9d51285d0e..46bebdd3d6 100644 --- a/mobile/lib/generated/intl/messages_pl.dart +++ b/mobile/lib/generated/intl/messages_pl.dart @@ -819,6 +819,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Grupuj pobliskie zdjęcia"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Nie śledzimy instalacji aplikacji. Pomogłyby nam, gdybyś powiedział/a nam, gdzie nas znalazłeś/aś!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1504,8 +1506,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Zaproponuj funkcje"), "support": MessageLookupByLibrary.simpleMessage("Wsparcie techniczne"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "Aby włączyć blokadę aplikacji, należy skonfigurować hasło urządzenia lub blokadę ekranu w ustawieniach Twojego systemu."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Synchronizacja zatrzymana"), diff --git a/mobile/lib/generated/intl/messages_pt.dart b/mobile/lib/generated/intl/messages_pt.dart index 6e1e0d8947..a628a8637b 100644 --- a/mobile/lib/generated/intl/messages_pt.dart +++ b/mobile/lib/generated/intl/messages_pt.dart @@ -818,6 +818,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("Agrupar fotos próximas"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Não rastreamos instalações do aplicativo. Seria útil se você nos contasse onde nos encontrou!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1506,8 +1508,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Sugerir recurso"), "support": MessageLookupByLibrary.simpleMessage("Suporte"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "Para ativar o bloqueio por deslizar, por favor ative um método de autenticação nas configurações do sistema do seu dispositivo."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Sincronização interrompida"), diff --git a/mobile/lib/generated/intl/messages_ru.dart b/mobile/lib/generated/intl/messages_ru.dart index 08becfa5fe..506d3726f5 100644 --- a/mobile/lib/generated/intl/messages_ru.dart +++ b/mobile/lib/generated/intl/messages_ru.dart @@ -812,6 +812,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Группировать фотографии рядом"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Будет полезно, если вы укажете, где нашли нас, так как мы не отслеживаем установки приложения!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1502,8 +1504,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Предложить идею"), "support": MessageLookupByLibrary.simpleMessage("Поддержка"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Синхронизация остановлена"), diff --git a/mobile/lib/generated/intl/messages_tr.dart b/mobile/lib/generated/intl/messages_tr.dart index d4821c86b5..481534b2c0 100644 --- a/mobile/lib/generated/intl/messages_tr.dart +++ b/mobile/lib/generated/intl/messages_tr.dart @@ -814,6 +814,8 @@ class MessageLookup extends MessageLookupByLibrary { "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage( "Yakındaki fotoğrafları gruplandır"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "Biz uygulama kurulumlarını takip etmiyoruz. Bizi nereden duyduğunuzdan bahsetmeniz bize çok yardımcı olacak!"), "hearUsWhereTitle": MessageLookupByLibrary.simpleMessage( @@ -1491,8 +1493,6 @@ class MessageLookup extends MessageLookupByLibrary { "suggestFeatures": MessageLookupByLibrary.simpleMessage("Özellik önerin"), "support": MessageLookupByLibrary.simpleMessage("Destek"), - "swipeLockEnablePreSteps": MessageLookupByLibrary.simpleMessage( - "To enable swipe lock, please setup device passcode or screen lock in your system settings."), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("Senkronizasyon durduruldu"), diff --git a/mobile/lib/generated/intl/messages_zh.dart b/mobile/lib/generated/intl/messages_zh.dart index 164d5b87f9..f46ffc9fc4 100644 --- a/mobile/lib/generated/intl/messages_zh.dart +++ b/mobile/lib/generated/intl/messages_zh.dart @@ -670,6 +670,8 @@ class MessageLookup extends MessageLookupByLibrary { "grantPermission": MessageLookupByLibrary.simpleMessage("授予权限"), "groupNearbyPhotos": MessageLookupByLibrary.simpleMessage("将附近的照片分组"), "guestView": MessageLookupByLibrary.simpleMessage("Guest view"), + "guestViewEnablePreSteps": MessageLookupByLibrary.simpleMessage( + "To enable guest view, please setup device passcode or screen lock in your system settings."), "hearUsExplanation": MessageLookupByLibrary.simpleMessage( "我们不跟踪应用程序安装情况。如果您告诉我们您是在哪里找到我们的，将会有所帮助！"), "hearUsWhereTitle": @@ -1212,8 +1214,6 @@ class MessageLookup extends MessageLookupByLibrary { "successfullyUnhid": MessageLookupByLibrary.simpleMessage("已成功取消隐藏"), "suggestFeatures": MessageLookupByLibrary.simpleMessage("建议新功能"), "support": MessageLookupByLibrary.simpleMessage("支持"), - "swipeLockEnablePreSteps": - MessageLookupByLibrary.simpleMessage("要启用滑动锁定，请在系统设置中设置设备密码或屏幕锁。"), "syncProgress": m62, "syncStopped": MessageLookupByLibrary.simpleMessage("同步已停止"), "syncing": MessageLookupByLibrary.simpleMessage("正在同步···"), diff --git a/mobile/lib/generated/l10n.dart b/mobile/lib/generated/l10n.dart index 7add93a337..7cb3716b4d 100644 --- a/mobile/lib/generated/l10n.dart +++ b/mobile/lib/generated/l10n.dart @@ -9125,16 +9125,6 @@ class S { ); } - /// `To enable swipe lock, please setup device passcode or screen lock in your system settings.` - String get swipeLockEnablePreSteps { - return Intl.message( - 'To enable swipe lock, please setup device passcode or screen lock in your system settings.', - name: 'swipeLockEnablePreSteps', - desc: '', - args: [], - ); - } - /// `Auto lock` String get autoLock { return Intl.message( @@ -9254,6 +9244,16 @@ class S { args: [], ); } + + /// `To enable guest view, please setup device passcode or screen lock in your system settings.` + String get guestViewEnablePreSteps { + return Intl.message( + 'To enable guest view, please setup device passcode or screen lock in your system settings.', + name: 'guestViewEnablePreSteps', + desc: '', + args: [], + ); + } } class AppLocalizationDelegate extends LocalizationsDelegate { diff --git a/mobile/lib/l10n/intl_cs.arb b/mobile/lib/l10n/intl_cs.arb index 2f2942c442..f5991dd0b0 100644 --- a/mobile/lib/l10n/intl_cs.arb +++ b/mobile/lib/l10n/intl_cs.arb @@ -53,5 +53,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_de.arb b/mobile/lib/l10n/intl_de.arb index 57f30fa51e..ceb5d28a44 100644 --- a/mobile/lib/l10n/intl_de.arb +++ b/mobile/lib/l10n/intl_de.arb @@ -1291,5 +1291,6 @@ "pleaseSelectQuickLinksToRemove": "Bitte wähle die zu entfernenden schnellen Links", "removePublicLinks": "Öffentliche Links entfernen", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Hiermit werden die öffentlichen Links aller ausgewählten schnellen Links entfernt.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_en.arb b/mobile/lib/l10n/intl_en.arb index e29bf9a14c..f7f842f493 100644 --- a/mobile/lib/l10n/intl_en.arb +++ b/mobile/lib/l10n/intl_en.arb @@ -1279,7 +1279,6 @@ "tooManyIncorrectAttempts": "Too many incorrect attempts", "videoInfo": "Video Info", "appLockDescription": "Choose between your device\\'s default lock screen and a custom lock screen with a PIN or password.", - "swipeLockEnablePreSteps": "To enable swipe lock, please setup device passcode or screen lock in your system settings.", "autoLock": "Auto lock", "immediately": "Immediately", "autoLockFeatureDescription": "Time after which the app locks after being put in the background", @@ -1291,5 +1290,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_es.arb b/mobile/lib/l10n/intl_es.arb index 280d87d0de..59affb44b2 100644 --- a/mobile/lib/l10n/intl_es.arb +++ b/mobile/lib/l10n/intl_es.arb @@ -1277,5 +1277,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_fr.arb b/mobile/lib/l10n/intl_fr.arb index 7e7c87caf2..17aa27bab6 100644 --- a/mobile/lib/l10n/intl_fr.arb +++ b/mobile/lib/l10n/intl_fr.arb @@ -1194,5 +1194,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_it.arb b/mobile/lib/l10n/intl_it.arb index 568c500b3e..45188038fd 100644 --- a/mobile/lib/l10n/intl_it.arb +++ b/mobile/lib/l10n/intl_it.arb @@ -1156,5 +1156,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_ko.arb b/mobile/lib/l10n/intl_ko.arb index 2f2942c442..f5991dd0b0 100644 --- a/mobile/lib/l10n/intl_ko.arb +++ b/mobile/lib/l10n/intl_ko.arb @@ -53,5 +53,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_nl.arb b/mobile/lib/l10n/intl_nl.arb index b7778c513e..a7105491da 100644 --- a/mobile/lib/l10n/intl_nl.arb +++ b/mobile/lib/l10n/intl_nl.arb @@ -1291,5 +1291,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_no.arb b/mobile/lib/l10n/intl_no.arb index 9c1aaeda09..2b6829f1f9 100644 --- a/mobile/lib/l10n/intl_no.arb +++ b/mobile/lib/l10n/intl_no.arb @@ -67,5 +67,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_pl.arb b/mobile/lib/l10n/intl_pl.arb index a13d440446..cc074a7bef 100644 --- a/mobile/lib/l10n/intl_pl.arb +++ b/mobile/lib/l10n/intl_pl.arb @@ -1291,5 +1291,6 @@ "pleaseSelectQuickLinksToRemove": "Prosimy wybrać szybkie linki do usunięcia", "removePublicLinks": "Usuń linki publiczne", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Spowoduje to usunięcie publicznych linków wszystkich zaznaczonych szybkich linków.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_pt.arb b/mobile/lib/l10n/intl_pt.arb index 7c9252873b..1259379354 100644 --- a/mobile/lib/l10n/intl_pt.arb +++ b/mobile/lib/l10n/intl_pt.arb @@ -1291,5 +1291,6 @@ "pleaseSelectQuickLinksToRemove": "Selecione links rápidos para remover", "removePublicLinks": "Remover link público", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Isto removerá links públicos de todos os links rápidos selecionados.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_ru.arb b/mobile/lib/l10n/intl_ru.arb index 428cdea90d..60dc6271af 100644 --- a/mobile/lib/l10n/intl_ru.arb +++ b/mobile/lib/l10n/intl_ru.arb @@ -1276,5 +1276,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_tr.arb b/mobile/lib/l10n/intl_tr.arb index ff66a5dfd1..0874270399 100644 --- a/mobile/lib/l10n/intl_tr.arb +++ b/mobile/lib/l10n/intl_tr.arb @@ -1288,5 +1288,6 @@ "pleaseSelectQuickLinksToRemove": "Please select quick links to remove", "removePublicLinks": "Remove public links", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "This will remove public links of all selected quick links.", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_zh.arb b/mobile/lib/l10n/intl_zh.arb index fac2a8f82b..2055ef0322 100644 --- a/mobile/lib/l10n/intl_zh.arb +++ b/mobile/lib/l10n/intl_zh.arb @@ -1291,5 +1291,6 @@ "pleaseSelectQuickLinksToRemove": "请选择要删除的快速链接", "removePublicLinks": "删除公开链接", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "这将删除所有选定的快速链接的公共链接。", - "guestView": "Guest view" + "guestView": "Guest view", + "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." } \ No newline at end of file diff --git a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart index c5af5deab4..29b71de3f9 100644 --- a/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart +++ b/mobile/lib/ui/viewer/actions/file_selection_actions_widget.dart @@ -4,6 +4,7 @@ import 'package:fast_base58/fast_base58.dart'; import "package:flutter/cupertino.dart"; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; +import "package:local_auth/local_auth.dart"; import "package:logging/logging.dart"; import "package:modal_bottom_sheet/modal_bottom_sheet.dart"; import 'package:photos/core/configuration.dart'; @@ -568,18 +569,26 @@ class _FileSelectionActionsWidgetState Future _onGuestViewClick() async { final List selectedFiles = widget.selectedFiles.files.toList(); - final page = DetailPage( - DetailPageConfiguration( - selectedFiles, - null, - 0, - "guest_view", - ), - ); - routeToPage(context, page, forceCustomPageRoute: true).ignore(); - WidgetsBinding.instance.addPostFrameCallback((_) { - Bus.instance.fire(GuestViewEvent(true, false)); - }); + if (await LocalAuthentication().isDeviceSupported()) { + final page = DetailPage( + DetailPageConfiguration( + selectedFiles, + null, + 0, + "guest_view", + ), + ); + routeToPage(context, page, forceCustomPageRoute: true).ignore(); + WidgetsBinding.instance.addPostFrameCallback((_) { + Bus.instance.fire(GuestViewEvent(true, false)); + }); + } else { + await showErrorDialog( + context, + S.of(context).noSystemLockFound, + S.of(context).guestViewEnablePreSteps, + ); + } widget.selectedFiles.clearAll(); } diff --git a/mobile/lib/ui/viewer/file/file_app_bar.dart b/mobile/lib/ui/viewer/file/file_app_bar.dart index e9221b2f72..8fbf260bca 100644 --- a/mobile/lib/ui/viewer/file/file_app_bar.dart +++ b/mobile/lib/ui/viewer/file/file_app_bar.dart @@ -419,7 +419,7 @@ class FileAppBarState extends State { await showErrorDialog( context, S.of(context).noSystemLockFound, - S.of(context).swipeLockEnablePreSteps, + S.of(context).guestViewEnablePreSteps, ); } } From 38c3e73638ca0ead67207f8f7ebc3c328b503666 Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Fri, 9 Aug 2024 20:18:28 +0530 Subject: [PATCH 128/211] fix: move panorama icon to middle --- mobile/lib/ui/viewer/file/detail_page.dart | 63 +++++++ .../lib/ui/viewer/file/file_bottom_bar.dart | 156 ++++++------------ .../viewer/file/panorama_viewer_screen.dart | 6 + 3 files changed, 120 insertions(+), 105 deletions(-) diff --git a/mobile/lib/ui/viewer/file/detail_page.dart b/mobile/lib/ui/viewer/file/detail_page.dart index 521a12ea69..87c282d2ba 100644 --- a/mobile/lib/ui/viewer/file/detail_page.dart +++ b/mobile/lib/ui/viewer/file/detail_page.dart @@ -12,6 +12,7 @@ import 'package:photos/core/errors.dart'; import "package:photos/core/event_bus.dart"; import "package:photos/events/file_swipe_lock_event.dart"; import "package:photos/generated/l10n.dart"; +import "package:photos/models/file/extensions/file_props.dart"; import 'package:photos/models/file/file.dart'; import "package:photos/models/file/file_type.dart"; import "package:photos/services/local_authentication_service.dart"; @@ -21,10 +22,12 @@ import "package:photos/ui/tools/editor/video_editor_page.dart"; import "package:photos/ui/viewer/file/file_app_bar.dart"; import "package:photos/ui/viewer/file/file_bottom_bar.dart"; import 'package:photos/ui/viewer/file/file_widget.dart'; +import "package:photos/ui/viewer/file/panorama_viewer_screen.dart"; import 'package:photos/ui/viewer/gallery/gallery.dart'; import 'package:photos/utils/dialog_util.dart'; import 'package:photos/utils/file_util.dart'; import 'package:photos/utils/navigation_util.dart'; +import "package:photos/utils/thumbnail_util.dart"; import 'package:photos/utils/toast_util.dart'; enum DetailPageMode { @@ -184,6 +187,48 @@ class _DetailPageState extends State { }, valueListenable: _selectedIndexNotifier, ), + ValueListenableBuilder( + valueListenable: _selectedIndexNotifier, + builder: (BuildContext context, int selectedIndex, _) { + if (_files![selectedIndex].isPanorama() == true) { + return ValueListenableBuilder( + valueListenable: _enableFullScreenNotifier, + builder: (context, value, child) { + return IgnorePointer( + ignoring: value, + child: AnimatedOpacity( + duration: const Duration(milliseconds: 200), + opacity: !value ? 1.0 : 0.0, + child: Align( + alignment: Alignment.center, + child: Tooltip( + message: S.of(context).panorama, + child: IconButton( + style: IconButton.styleFrom( + backgroundColor: const Color(0xAA252525), + fixedSize: const Size(44, 44), + ), + icon: const Icon( + Icons.threesixty, + color: Colors.white, + size: 26, + ), + onPressed: () async { + await openPanoramaViewerPage( + _files![selectedIndex], + ); + }, + ), + ), + ), + ), + ); + }, + ); + } + return const SizedBox(); + }, + ), ], ), ), @@ -191,6 +236,24 @@ class _DetailPageState extends State { ); } + Future openPanoramaViewerPage(EnteFile file) async { + final fetchedFile = await getFile(file); + if (fetchedFile == null) { + return; + } + final fetchedThumbnail = await getThumbnail(file); + Navigator.of(context).push( + MaterialPageRoute( + builder: (_) { + return PanoramaViewerScreen( + file: fetchedFile, + thumbnail: fetchedThumbnail, + ); + }, + ), + ).ignore(); + } + Widget _buildPageView(BuildContext context) { return PageView.builder( clipBehavior: Clip.none, diff --git a/mobile/lib/ui/viewer/file/file_bottom_bar.dart b/mobile/lib/ui/viewer/file/file_bottom_bar.dart index 9b60748386..0d08abcb96 100644 --- a/mobile/lib/ui/viewer/file/file_bottom_bar.dart +++ b/mobile/lib/ui/viewer/file/file_bottom_bar.dart @@ -17,12 +17,9 @@ import "package:photos/theme/colors.dart"; import "package:photos/theme/ente_theme.dart"; import "package:photos/ui/actions/file/file_actions.dart"; import 'package:photos/ui/collections/collection_action_sheet.dart'; -import "package:photos/ui/viewer/file/panorama_viewer_screen.dart"; import 'package:photos/utils/delete_file_util.dart'; -import "package:photos/utils/file_util.dart"; import "package:photos/utils/panorama_util.dart"; import 'package:photos/utils/share_util.dart'; -import "package:photos/utils/thumbnail_util.dart"; class FileBottomBar extends StatefulWidget { final EnteFile file; @@ -198,92 +195,59 @@ class FileBottomBarState extends State { curve: Curves.easeInOut, child: Align( alignment: Alignment.bottomCenter, - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - if (widget.file.isPanorama() == true) - Align( - alignment: Alignment.centerRight, - child: Tooltip( - message: S.of(context).panorama, - child: Padding( - padding: const EdgeInsets.only( - top: 12, - bottom: 12, - right: 20, - ), - child: IconButton( - style: IconButton.styleFrom( - backgroundColor: const Color(0xFF252525), - fixedSize: const Size(44, 44), - ), - icon: const Icon( - Icons.vrpano_outlined, - color: Colors.white, - size: 26, - ), - onPressed: () async { - await openPanoramaViewerPage(widget.file); - }, - ), - ), - ), - ), - Container( - decoration: BoxDecoration( - gradient: LinearGradient( - begin: Alignment.topCenter, - end: Alignment.bottomCenter, - colors: [ - Colors.transparent, - Colors.black.withOpacity(0.6), - Colors.black.withOpacity(0.72), - ], - stops: const [0, 0.8, 1], - ), - ), - child: Padding( - padding: EdgeInsets.only(bottom: safeAreaBottomPadding), - child: Column( - mainAxisSize: MainAxisSize.min, - children: [ - widget.file.caption?.isNotEmpty ?? false - ? Padding( - padding: const EdgeInsets.fromLTRB( - 16, - 12, - 16, - 0, - ), - child: GestureDetector( - onTap: () async { - await _displayDetails(widget.file); - await Future.delayed( - const Duration(milliseconds: 500), - ); //Waiting for some time till the caption gets updated in db if the user closes the bottom sheet without pressing 'done' - safeRefresh(); - }, - child: Text( - widget.file.caption!, - overflow: TextOverflow.ellipsis, - maxLines: 1, - style: getEnteTextTheme(context) - .mini - .copyWith(color: textBaseDark), - textAlign: TextAlign.center, - ), - ), - ) - : const SizedBox.shrink(), - Row( - mainAxisAlignment: MainAxisAlignment.spaceAround, - children: children, - ), - ], + child: Container( + decoration: BoxDecoration( + gradient: LinearGradient( + begin: Alignment.topCenter, + end: Alignment.bottomCenter, + colors: [ + Colors.transparent, + Colors.black.withOpacity(0.6), + Colors.black.withOpacity(0.72), + ], + stops: const [0, 0.8, 1], + ), + ), + child: Padding( + padding: EdgeInsets.only(bottom: safeAreaBottomPadding), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + widget.file.caption?.isNotEmpty ?? false + ? Padding( + padding: const EdgeInsets.fromLTRB( + 16, + 12, + 16, + 0, + ), + child: GestureDetector( + onTap: () async { + await _displayDetails(widget.file); + await Future.delayed( + const Duration(milliseconds: 500), + ); //Waiting for some time till the caption gets updated in db if the user closes the bottom sheet without pressing 'done' + safeRefresh(); + }, + child: Text( + widget.file.caption!, + overflow: TextOverflow.ellipsis, + maxLines: 1, + style: getEnteTextTheme(context) + .mini + .copyWith(color: textBaseDark), + textAlign: TextAlign.center, + ), + ), + ) + : const SizedBox.shrink(), + Row( + mainAxisAlignment: MainAxisAlignment.spaceAround, + children: children, ), - ), + ], ), - ], + ), ), ), ), @@ -292,24 +256,6 @@ class FileBottomBarState extends State { ); } - Future openPanoramaViewerPage(EnteFile file) async { - final fetchedFile = await getFile(file); - if (fetchedFile == null) { - return; - } - final fetchedThumbnail = await getThumbnail(file); - Navigator.of(context).push( - MaterialPageRoute( - builder: (_) { - return PanoramaViewerScreen( - file: fetchedFile, - thumbnail: fetchedThumbnail, - ); - }, - ), - ).ignore(); - } - Future _showSingleFileDeleteSheet(EnteFile file) async { await showSingleFileDeleteSheet( context, diff --git a/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart b/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart index c9437b5ab6..27dedfcc60 100644 --- a/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart +++ b/mobile/lib/ui/viewer/file/panorama_viewer_screen.dart @@ -36,6 +36,12 @@ class _PanoramaViewerScreenState extends State { super.initState(); } + @override + void dispose() { + SystemChrome.setEnabledSystemUIMode(SystemUiMode.edgeToEdge); + super.dispose(); + } + void initTimer() { timer = Timer(const Duration(seconds: 5), () { setState(() { From e7e74c17f8a2d69287a2e7eedf4880cba2e65dc7 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 20:23:46 +0530 Subject: [PATCH 129/211] Fix debug flow --- web/apps/photos/src/services/searchService.ts | 3 ++- web/packages/new/photos/services/ml/index.ts | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/searchService.ts b/web/apps/photos/src/services/searchService.ts index 97721a13ad..c469c4eec5 100644 --- a/web/apps/photos/src/services/searchService.ts +++ b/web/apps/photos/src/services/searchService.ts @@ -419,13 +419,14 @@ function convertSuggestionToSearchQuery(option: Suggestion): Search { async function getAllPeople(limit: number = undefined) { if (!(await wipClusterEnable())) return []; - if (Math.random() < 0.01) { + if (process.env.NEXT_PUBLIC_ENTE_WIP_CL_FETCH) { const entityKey = await getEntityKey("person" as EntityType); const peopleR = await personDiff(entityKey.data); const r = peopleR.length; log.debug(() => ["people", peopleR]); if (r) return []; + return []; } let people: Array = []; // await mlIDbStorage.getAllPeople(); diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 7f44fa2010..9355e7d24b 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -317,7 +317,6 @@ let last: Person[] | undefined; export const wipClusterEnable = async () => { if (!isDevBuild || !(await isInternalUser())) return false; if (!process.env.NEXT_PUBLIC_ENTE_WIP_CL) return false; - if (last) return false; return true; }; From 5d28f75c1a37188b43b0f24f4548ed8b7bd8c1db Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Fri, 9 Aug 2024 20:30:03 +0530 Subject: [PATCH 130/211] Dedup --- web/apps/photos/src/services/entityService.ts | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/web/apps/photos/src/services/entityService.ts b/web/apps/photos/src/services/entityService.ts index be8a1e3a7f..120058f265 100644 --- a/web/apps/photos/src/services/entityService.ts +++ b/web/apps/photos/src/services/entityService.ts @@ -15,30 +15,6 @@ import { } from "types/entity"; import { getLatestVersionEntities } from "utils/entity"; -/** - * The maximum number of items to fetch in a single diff - * - * [Note: Limit of returned items in /diff requests] - * - * The various GET /diff API methods, which tell the client what all has changed - * since a timestamp (provided by the client) take a limit parameter. - * - * These diff API calls return all items whose updated at is greater - * (non-inclusive) than the timestamp we provide. So there is no mechanism for - * pagination of items which have the exact same updated at. - * - * Conceptually, it may happen that there are more items than the limit we've - * provided, but there are practical safeguards. - * - * For file diff, the limit is advisory, and remote may return less, equal or - * more items than the provided limit. The scenario where it returns more is - * when more files than the limit have the same updated at. Theoretically it - * would make the diff response unbounded, however in practice file - * modifications themselves are all batched. Even if the user were to select all - * the files in their library and updates them all in one go in the UI, their - * client app is required to use batched API calls to make those updates, and - * each of those batches would get distinct updated at. - */ const DIFF_LIMIT = 500; const ENTITY_TABLES: Record = { From 62e27916b76f26b2a4841ce61fb93bdaf03e77a2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 09:09:50 +0530 Subject: [PATCH 131/211] lf --- web/packages/new/photos/services/user-entity.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index c6cca31cc6..8641fd3ceb 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -125,11 +125,8 @@ export const userEntityDiff = async ( */ export const personDiff = async (entityKeyB64: string) => { const entities = await userEntityDiff("person", 0, entityKeyB64); - return Promise.all( - entities.map(async ({ data }) => { - if (!data) return undefined; - // return JSON.parse(await gunzip(data)) as unknown; - return JSON.parse(new TextDecoder().decode(data)) as unknown; - }), - ); + return entities.map(({ data }) => { + if (!data) return undefined; + return JSON.parse(new TextDecoder().decode(data)) as unknown; + }); }; From 93c5825364b8daff2a570a4a004d0164f41c5f17 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 08:55:16 +0530 Subject: [PATCH 132/211] Add MobileCLIP URLs --- desktop/src/main/services/ml-worker.ts | 32 ++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts index f4b9221f64..d6e61eeb7d 100644 --- a/desktop/src/main/services/ml-worker.ts +++ b/desktop/src/main/services/ml-worker.ts @@ -201,9 +201,15 @@ const createInferenceSession = async (modelPath: string) => { }); }; +// TODO-ML: Remove me +// const cachedCLIPImageSessionOAI = makeCachedInferenceSession( +// "clip-image-vit-32-float32.onnx", +// 351468764 /* 335 MB */, +// ); + const cachedCLIPImageSession = makeCachedInferenceSession( - "clip-image-vit-32-float32.onnx", - 351468764 /* 335.2 MB */, + "mobileclip_s2_image.onnx", + 143061211 /* 143 MB */, ); /** @@ -223,9 +229,27 @@ export const computeCLIPImageEmbedding = async (input: Float32Array) => { return ensure(results.output).data as Float32Array; }; +// TODO-ML: Remove me +// const cachedCLIPTextSessionOAIQ = makeCachedInferenceSession( +// "clip-text-vit-32-uint8.onnx", +// 64173509 /* 61 MB */, +// ); + +// TODO-ML: Remove me +// const cachedCLIPTextSessionOAI = makeCachedInferenceSession( +// "clip-text-vit-32-float32-int32.onnx", +// 254069585 /* 254 MB */, +// ); + +// TODO-ML: Remove me +// const cachedCLIPTextSession = makeCachedInferenceSession( +// "mobileclip_s2_text.onnx", +// 253895732 /* 253 MB */, +// ); + const cachedCLIPTextSession = makeCachedInferenceSession( - "clip-text-vit-32-uint8.onnx", - 64173509 /* 61.2 MB */, + "mobileclip_s2_text_int32.onnx", + 253895600 /* 253 MB */, ); let _tokenizer: Tokenizer | undefined; From 5ce8d9838fa56c324bc17b16ddd1b818cc620e80 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 09:48:07 +0530 Subject: [PATCH 133/211] 224 => 256 https://github.com/apple/ml-mobileclip/blob/main/mobileclip/configs/mobileclip_s2.json --- desktop/src/main/services/ml-worker.ts | 2 +- web/packages/new/photos/services/ml/clip.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts index d6e61eeb7d..40c1c5fb5e 100644 --- a/desktop/src/main/services/ml-worker.ts +++ b/desktop/src/main/services/ml-worker.ts @@ -220,7 +220,7 @@ const cachedCLIPImageSession = makeCachedInferenceSession( export const computeCLIPImageEmbedding = async (input: Float32Array) => { const session = await cachedCLIPImageSession(); const feeds = { - input: new ort.Tensor("float32", input, [1, 3, 224, 224]), + input: new ort.Tensor("float32", input, [1, 3, 256, 256]), }; const t = Date.now(); const results = await session.run(feeds); diff --git a/web/packages/new/photos/services/ml/clip.ts b/web/packages/new/photos/services/ml/clip.ts index 78eff1c04d..c61cb5b535 100644 --- a/web/packages/new/photos/services/ml/clip.ts +++ b/web/packages/new/photos/services/ml/clip.ts @@ -120,8 +120,7 @@ const computeEmbedding = async ( * Convert {@link imageData} into the format that the CLIP model expects. */ const convertToCLIPInput = (imageData: ImageData) => { - const requiredWidth = 224; - const requiredHeight = 224; + const [requiredWidth, requiredHeight] = [256, 256]; const mean = [0.48145466, 0.4578275, 0.40821073] as const; const std = [0.26862954, 0.26130258, 0.27577711] as const; From b503f7599952947b41a76fa366951030a6ddabc1 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 13:09:23 +0530 Subject: [PATCH 134/211] Don't need the mean/std --- web/packages/new/photos/services/ml/clip.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/services/ml/clip.ts b/web/packages/new/photos/services/ml/clip.ts index c61cb5b535..1cd14047f4 100644 --- a/web/packages/new/photos/services/ml/clip.ts +++ b/web/packages/new/photos/services/ml/clip.ts @@ -122,8 +122,10 @@ const computeEmbedding = async ( const convertToCLIPInput = (imageData: ImageData) => { const [requiredWidth, requiredHeight] = [256, 256]; - const mean = [0.48145466, 0.4578275, 0.40821073] as const; - const std = [0.26862954, 0.26130258, 0.27577711] as const; + // const mean = [0.48145466, 0.4578275, 0.40821073] as const; + const mean = [0, 0, 0] as const; + // const std = [0.26862954, 0.26130258, 0.27577711] as const; + const std = [1, 1, 1] as const; const { width, height, data: pixelData } = imageData; From 1f28fdada2bd249bbabffcf4f02cf37596ec5385 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 13:11:02 +0530 Subject: [PATCH 135/211] Bilinear --- web/packages/new/photos/services/ml/clip.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/services/ml/clip.ts b/web/packages/new/photos/services/ml/clip.ts index 1cd14047f4..4c19e0395b 100644 --- a/web/packages/new/photos/services/ml/clip.ts +++ b/web/packages/new/photos/services/ml/clip.ts @@ -1,7 +1,7 @@ import type { ElectronMLWorker } from "@/base/types/ipc"; import type { ImageBitmapAndData } from "./blob"; import { clipIndexes } from "./db"; -import { pixelRGBBicubic } from "./image"; +import { pixelRGBBilinear } from "./image"; import { dotProduct, norm } from "./math"; import type { CLIPMatches } from "./worker-types"; @@ -145,7 +145,7 @@ const convertToCLIPInput = (imageData: ImageData) => { const cOffsetB = 2 * requiredHeight * requiredWidth; // ChannelOffsetBlue for (let h = 0 + heightOffset; h < scaledHeight - heightOffset; h++) { for (let w = 0 + widthOffset; w < scaledWidth - widthOffset; w++) { - const { r, g, b } = pixelRGBBicubic( + const { r, g, b } = pixelRGBBilinear( w / scale, h / scale, pixelData, From 5bbc2615e432d4190519edb0c3249e4e002d038f Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 13:39:14 +0530 Subject: [PATCH 136/211] Tune the threshold for MobileCLIP Experimentation. - 0.15 was noisy - 0.23 was too strict --- web/packages/new/photos/services/ml/clip.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/new/photos/services/ml/clip.ts b/web/packages/new/photos/services/ml/clip.ts index 4c19e0395b..e0d1211fbe 100644 --- a/web/packages/new/photos/services/ml/clip.ts +++ b/web/packages/new/photos/services/ml/clip.ts @@ -190,5 +190,5 @@ export const clipMatches = async ( // This code is on the hot path, so these optimizations help. [fileID, dotProduct(embedding, textEmbedding)] as const, ); - return new Map(items.filter(([, score]) => score >= 0.23)); + return new Map(items.filter(([, score]) => score >= 0.2)); }; From 72bce123a5fc6b32e208086aad3c7025e2428992 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 13:42:26 +0530 Subject: [PATCH 137/211] Cleanup --- desktop/src/main/services/ml-worker.ts | 26 +------------------------- 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts index 40c1c5fb5e..68a8eb8349 100644 --- a/desktop/src/main/services/ml-worker.ts +++ b/desktop/src/main/services/ml-worker.ts @@ -201,12 +201,6 @@ const createInferenceSession = async (modelPath: string) => { }); }; -// TODO-ML: Remove me -// const cachedCLIPImageSessionOAI = makeCachedInferenceSession( -// "clip-image-vit-32-float32.onnx", -// 351468764 /* 335 MB */, -// ); - const cachedCLIPImageSession = makeCachedInferenceSession( "mobileclip_s2_image.onnx", 143061211 /* 143 MB */, @@ -229,24 +223,6 @@ export const computeCLIPImageEmbedding = async (input: Float32Array) => { return ensure(results.output).data as Float32Array; }; -// TODO-ML: Remove me -// const cachedCLIPTextSessionOAIQ = makeCachedInferenceSession( -// "clip-text-vit-32-uint8.onnx", -// 64173509 /* 61 MB */, -// ); - -// TODO-ML: Remove me -// const cachedCLIPTextSessionOAI = makeCachedInferenceSession( -// "clip-text-vit-32-float32-int32.onnx", -// 254069585 /* 254 MB */, -// ); - -// TODO-ML: Remove me -// const cachedCLIPTextSession = makeCachedInferenceSession( -// "mobileclip_s2_text.onnx", -// 253895732 /* 253 MB */, -// ); - const cachedCLIPTextSession = makeCachedInferenceSession( "mobileclip_s2_text_int32.onnx", 253895600 /* 253 MB */, @@ -294,7 +270,7 @@ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => { const cachedFaceDetectionSession = makeCachedInferenceSession( "yolov5s_face_640_640_dynamic.onnx", - 30762872 /* 29.3 MB */, + 30762872 /* 29 MB */, ); /** From ac8a5b491d0f02d349d21c1f6538d06e0445afb5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 13:45:03 +0530 Subject: [PATCH 138/211] Update refs --- desktop/src/main/services/ml-worker.ts | 4 ++-- web/packages/new/photos/services/ml/clip.ts | 18 +++++++----------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/desktop/src/main/services/ml-worker.ts b/desktop/src/main/services/ml-worker.ts index 68a8eb8349..cd9a3e0404 100644 --- a/desktop/src/main/services/ml-worker.ts +++ b/desktop/src/main/services/ml-worker.ts @@ -209,7 +209,7 @@ const cachedCLIPImageSession = makeCachedInferenceSession( /** * Compute CLIP embeddings for an image. * - * The embeddings are computed using ONNX runtime, with CLIP as the model. + * The embeddings are computed using ONNX runtime, with MobileCLIP as the model. */ export const computeCLIPImageEmbedding = async (input: Float32Array) => { const session = await cachedCLIPImageSession(); @@ -237,7 +237,7 @@ const getTokenizer = () => { /** * Compute CLIP embeddings for an text snippet. * - * The embeddings are computed using ONNX runtime, with CLIP as the model. + * The embeddings are computed using ONNX runtime, with MobileCLIP as the model. */ export const computeCLIPTextEmbeddingIfAvailable = async (text: string) => { const sessionOrSkip = await Promise.race([ diff --git a/web/packages/new/photos/services/ml/clip.ts b/web/packages/new/photos/services/ml/clip.ts index e0d1211fbe..b226ef10cb 100644 --- a/web/packages/new/photos/services/ml/clip.ts +++ b/web/packages/new/photos/services/ml/clip.ts @@ -39,8 +39,9 @@ export const clipIndexingVersion = 1; * initial launch of this feature using the GGML runtime. * * Since the initial launch, we've switched over to another runtime, - * [ONNX](https://onnxruntime.ai) and have made other implementation changes, - * but the overall gist remains the same. + * [ONNX](https://onnxruntime.ai), started using Apple's + * [MobileCLIP](https://github.com/apple/ml-mobileclip/) as the model and have + * made other implementation changes, but the overall gist remains the same. * * Note that we don't train the neural network - we only use one of the publicly * available pre-trained neural networks for inference. These neural networks @@ -117,16 +118,11 @@ const computeEmbedding = async ( }; /** - * Convert {@link imageData} into the format that the CLIP model expects. + * Convert {@link imageData} into the format that the MobileCLIP model expects. */ const convertToCLIPInput = (imageData: ImageData) => { const [requiredWidth, requiredHeight] = [256, 256]; - // const mean = [0.48145466, 0.4578275, 0.40821073] as const; - const mean = [0, 0, 0] as const; - // const std = [0.26862954, 0.26130258, 0.27577711] as const; - const std = [1, 1, 1] as const; - const { width, height, data: pixelData } = imageData; // Maintain aspect ratio. @@ -152,9 +148,9 @@ const convertToCLIPInput = (imageData: ImageData) => { width, height, ); - clipInput[pi] = (r / 255.0 - mean[0]) / std[0]; - clipInput[pi + cOffsetG] = (g / 255.0 - mean[1]) / std[1]; - clipInput[pi + cOffsetB] = (b / 255.0 - mean[2]) / std[2]; + clipInput[pi] = r / 255.0; + clipInput[pi + cOffsetG] = g / 255.0; + clipInput[pi + cOffsetB] = b / 255.0; pi++; } } From 42c3482423676161b85d7d8376f15447d60bc80b Mon Sep 17 00:00:00 2001 From: vktr2b <65308076+vktr2b@users.noreply.github.com> Date: Sat, 10 Aug 2024 11:53:04 +0200 Subject: [PATCH 139/211] Added & Improved Logos (#2642) ## Description Added logos for: - Infomaniak - OpenObserve - Vikunja - SMSPool - SMTP2GO Improvements / Minor Changes - Brave Icon config - Crypto.com Icon config --- .../custom-icons/_data/custom-icons.json | 43 ++++++++++++++++++- auth/assets/custom-icons/icons/infomaniak.svg | 5 +++ .../custom-icons/icons/open_observe.svg | 39 +++++++++++++++++ .../custom-icons/icons/sms_pool_net.svg | 10 +++++ auth/assets/custom-icons/icons/smtp2go.svg | 1 + auth/assets/custom-icons/icons/vikunja.svg | 12 ++++++ 6 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 auth/assets/custom-icons/icons/infomaniak.svg create mode 100644 auth/assets/custom-icons/icons/open_observe.svg create mode 100644 auth/assets/custom-icons/icons/sms_pool_net.svg create mode 100644 auth/assets/custom-icons/icons/smtp2go.svg create mode 100644 auth/assets/custom-icons/icons/vikunja.svg diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index 80f81b2887..36e26a0aa6 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -84,7 +84,12 @@ }, { "title": "Brave Creators", - "slug": "brave_creators" + "slug": "brave_creators", + "altNames":[ + "Brave", + "Brave Rewards", + "Brave Browser" + ] }, { "title": "Bybit" @@ -125,7 +130,13 @@ "title": "Crowdpear" }, { - "title": "crypto" + "title": "Crypto.com", + "slug": "crypto", + "altNames": [ + "crypto", + "Crypto.com", + "Crypto com" + ] }, { "title": "DCS", @@ -220,6 +231,10 @@ "title": "IceDrive", "slug": "Icedrive" }, + { + "titile": "Infomaniak", + "slug": "infomaniak" + }, { "title": "ING" }, @@ -373,6 +388,14 @@ { "title": "Odido" }, + { + "titile": "OpenObserve", + "slug": "open_observe", + "altNames":[ + "openobserve.ai", + "openobserve ai" + ] + }, { "title": "okx", "hex": "000000" }, { @@ -477,6 +500,18 @@ { "title": "Skinport" }, + { + "title": "SMSPool", + "slug": "sms_pool_net", + "altNames": [ + "smspool.net", + "smspool net" + ] + }, + { + "title": "SMTP2GO", + "slug": "smtp2go" + }, { "title": "Snapchat" }, @@ -563,6 +598,10 @@ "slug": "uphold", "hex": "6FE68A" }, + { + "titile": "Vikunja", + "slug": "vikunja" + }, { "title": "WHMCS" }, diff --git a/auth/assets/custom-icons/icons/infomaniak.svg b/auth/assets/custom-icons/icons/infomaniak.svg new file mode 100644 index 0000000000..d79e699e4a --- /dev/null +++ b/auth/assets/custom-icons/icons/infomaniak.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/auth/assets/custom-icons/icons/open_observe.svg b/auth/assets/custom-icons/icons/open_observe.svg new file mode 100644 index 0000000000..2c50ad73a3 --- /dev/null +++ b/auth/assets/custom-icons/icons/open_observe.svg @@ -0,0 +1,39 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/auth/assets/custom-icons/icons/sms_pool_net.svg b/auth/assets/custom-icons/icons/sms_pool_net.svg new file mode 100644 index 0000000000..a57bc71226 --- /dev/null +++ b/auth/assets/custom-icons/icons/sms_pool_net.svg @@ -0,0 +1,10 @@ + + + + + + + + + + diff --git a/auth/assets/custom-icons/icons/smtp2go.svg b/auth/assets/custom-icons/icons/smtp2go.svg new file mode 100644 index 0000000000..5cd0c1b6db --- /dev/null +++ b/auth/assets/custom-icons/icons/smtp2go.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/auth/assets/custom-icons/icons/vikunja.svg b/auth/assets/custom-icons/icons/vikunja.svg new file mode 100644 index 0000000000..53176d66ef --- /dev/null +++ b/auth/assets/custom-icons/icons/vikunja.svg @@ -0,0 +1,12 @@ + + + + + + + + + + + + From 23da3b1c8495ada63175cbd7e0eb0b67d4865d37 Mon Sep 17 00:00:00 2001 From: ludespeedny Date: Sat, 10 Aug 2024 05:53:35 -0400 Subject: [PATCH 140/211] Custom icons for auth (#2646) Added "enom" as a custom icon and added the entry in custom-icons.json file. --- auth/assets/custom-icons/_data/custom-icons.json | 9 +++++++-- auth/assets/custom-icons/icons/Enom.svg | 10 ++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 auth/assets/custom-icons/icons/Enom.svg diff --git a/auth/assets/custom-icons/_data/custom-icons.json b/auth/assets/custom-icons/_data/custom-icons.json index 36e26a0aa6..7bae017eb1 100644 --- a/auth/assets/custom-icons/_data/custom-icons.json +++ b/auth/assets/custom-icons/_data/custom-icons.json @@ -175,6 +175,10 @@ "title": "ente", "hex": "1DB954" }, + { + "title": "enom" + }, + { "title": "Epic Games", "slug": "epic_games", @@ -249,8 +253,9 @@ "title": "INWX" }, { - "title": "Itch.io", - "slug": "itch_io" + "title": "Itch", + "slug": "itch_io", + "hex": "e7685e" }, { "title": "IVPN", diff --git a/auth/assets/custom-icons/icons/Enom.svg b/auth/assets/custom-icons/icons/Enom.svg new file mode 100644 index 0000000000..9c8ff617ae --- /dev/null +++ b/auth/assets/custom-icons/icons/Enom.svg @@ -0,0 +1,10 @@ + + + + + + + + + + From 18b0bd4996e26f649f196ad67e83a9828ed491b0 Mon Sep 17 00:00:00 2001 From: Prateek Sunal Date: Sat, 10 Aug 2024 16:00:31 +0530 Subject: [PATCH 141/211] fix(auth): allow without backups for windows --- auth/lib/onboarding/view/onboarding_page.dart | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/auth/lib/onboarding/view/onboarding_page.dart b/auth/lib/onboarding/view/onboarding_page.dart index a0bc6fb661..dff1fbc179 100644 --- a/auth/lib/onboarding/view/onboarding_page.dart +++ b/auth/lib/onboarding/view/onboarding_page.dart @@ -219,9 +219,10 @@ class _OnboardingPageState extends State { } Future _optForOfflineMode() async { - bool canCheckBio = Platform.isMacOS || Platform.isLinux - ? true - : await LocalAuthentication().canCheckBiometrics; + bool canCheckBio = Platform.isMacOS || + Platform.isLinux || + Platform.isWindows || + await LocalAuthentication().canCheckBiometrics; if (!canCheckBio) { showToast( context, From a029b168516f33f87dce6c3f7472e6828a3925a3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 11:31:30 +0530 Subject: [PATCH 142/211] Transparent but handrolled proxy --- web/packages/base/crypto/ente.ts | 21 ++++++++++++++++++- .../shared/crypto/internal/crypto.worker.ts | 4 ++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index f1dedb94e0..74dfb800bb 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -17,7 +17,11 @@ * 2. internal/libsodium.ts (wrappers over libsodium) * 3. libsodium (JS bindings). */ +import ComlinkCryptoWorker from "@ente/shared/crypto"; import * as libsodium from "@ente/shared/crypto/internal/libsodium"; +import { inWorker } from "../env"; + +const cryptoWorker = () => ComlinkCryptoWorker.getInstance(); /** * Encrypt arbitrary data associated with an Ente object (file, collection, @@ -200,7 +204,22 @@ export const decryptMetadata = async ( * decrypted data as a JSON string and instead just returns the raw decrypted * bytes that we got. */ -export const decryptMetadataBytes = async ( +export const decryptMetadataBytes = ( + encryptedDataB64: string, + decryptionHeaderB64: string, + keyB64: string, +) => + inWorker() + ? decryptMetadataBytesI(encryptedDataB64, decryptionHeaderB64, keyB64) + : cryptoWorker().then((cw) => + cw.decryptMetadataBytes( + encryptedDataB64, + decryptionHeaderB64, + keyB64, + ), + ); + +export const decryptMetadataBytesI = async ( encryptedDataB64: string, decryptionHeaderB64: string, keyB64: string, diff --git a/web/packages/shared/crypto/internal/crypto.worker.ts b/web/packages/shared/crypto/internal/crypto.worker.ts index 356bde8580..15b6b15b27 100644 --- a/web/packages/shared/crypto/internal/crypto.worker.ts +++ b/web/packages/shared/crypto/internal/crypto.worker.ts @@ -40,6 +40,10 @@ export class DedicatedCryptoWorker { ); } + async decryptMetadataBytes(a: string, b: string, c: string) { + return ente.decryptMetadataBytesI(a, b, c); + } + async decryptFile(fileData: Uint8Array, header: Uint8Array, key: string) { return libsodium.decryptChaCha(fileData, header, key); } From d13c23f2d8e4ea8896d7da33b6519a61284eb257 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 11:48:57 +0530 Subject: [PATCH 143/211] Doc --- web/packages/base/crypto/ente.ts | 51 ++++++++++++++----- .../shared/crypto/internal/crypto.worker.ts | 11 +--- .../shared/crypto/internal/libsodium.ts | 4 +- 3 files changed, 43 insertions(+), 23 deletions(-) diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index 74dfb800bb..e28583137f 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -3,19 +3,44 @@ * * [Note: Crypto code hierarchy] * - * The functions in this file (base/crypto/ente.ts) are are thin wrappers over - * the (thin-) wrappers in internal/libsodium.ts. The main difference is that - * these functions don't talk in terms of the crypto algorithms, but rather in - * terms the higher-level Ente specific goal we are trying to accomplish. - * - * Some of these are also exposed via the web worker in - * internal/crypto.worker.ts. The web worker variants should be used when we - * need to perform these operations from the main thread, so that the UI remains - * responsive while the potentially CPU-intensive encryption etc happens. - * - * 1. ente.ts or crypto.worker.ts (high level, Ente specific). - * 2. internal/libsodium.ts (wrappers over libsodium) - * 3. libsodium (JS bindings). + * 1. ente.ts or crypto.worker.ts (high level, Ente specific). + * 2. internal/libsodium.ts (wrappers over libsodium) + * 3. libsodium (JS bindings). + * + * Our cryptography primitives are provided by libsodium, specifically, its + * JavaScript bindings ("libsodium-wrappers"). This is the lowest layer. + * + * Direct usage of "libsodium-wrappers" is restricted to + * `crypto/internal/libsodium.ts`. This is the next higher layer, and the first + * one that our code should directly use. Usually the functions in this file are + * thin wrappers over the raw libsodium APIs, with a bit of massaging or + * book-keeping. They also ensure that sodium.ready has been called before + * accessing libsodium's APIs, thus all the functions it exposes are async. + * + * The final layer is this file, `crypto/ente.ts`. These are usually thin + * wrappers themselves over functions exposed by `internal/libsodium.ts`, but + * the difference is that the functions in ente.ts don't talk in terms of the + * crypto algorithms, but rather in terms the higher-level Ente specific goal we + * are trying to accomplish. + * + * There is an additional actor in the play. Cryptographic operations are CPU + * intensive and would cause the UI to stutter if used directly on the main + * thread. To keep the UI smooth, we instead want to run them in a web worker. + * However, sometimes we already _are_ running in a web worker, and delegating + * to another worker is wasteful. + * + * To handle both these scenario, each function in this file is split into the + * external API, and the underlying implementation (denoted by an "I" suffix). + * The external API functions check to see if we're already in a web worker, and + * if so directly invoke the implementation. Otherwise the call the sibling + * function in a shared "crypto" web worker (which then invokes the + * implementation, but this time in the context of a web worker). + * + * Some older code directly calls the functions in the shared crypto.worker.ts, + * but that should be avoided since it makes the code not behave the way we want + * when we're already in a web worker. There are exceptions to this + * recommendation though (in circumstances where we create more crypto workers + * instead of using the shared one). */ import ComlinkCryptoWorker from "@ente/shared/crypto"; import * as libsodium from "@ente/shared/crypto/internal/libsodium"; diff --git a/web/packages/shared/crypto/internal/crypto.worker.ts b/web/packages/shared/crypto/internal/crypto.worker.ts index 15b6b15b27..319d75c57d 100644 --- a/web/packages/shared/crypto/internal/crypto.worker.ts +++ b/web/packages/shared/crypto/internal/crypto.worker.ts @@ -1,6 +1,6 @@ import * as ente from "@/base/crypto/ente"; import * as libsodium from "@ente/shared/crypto/internal/libsodium"; -import * as Comlink from "comlink"; +import { expose } from "comlink"; import type { StateAddress } from "libsodium-wrappers"; /** @@ -8,13 +8,6 @@ import type { StateAddress } from "libsodium-wrappers"; * specific layer (base/crypto/ente.ts) or the internal libsodium layer * (internal/libsodium.ts). * - * Use these when running on the main thread, since running these in a web - * worker allows us to use potentially CPU-intensive crypto operations from the - * main thread without stalling the UI. - * - * If the code that needs this functionality is already running in the context - * of a web worker, then use the underlying functions directly. - * * See: [Note: Crypto code hierarchy]. * * Note: Keep these methods logic free. They are meant to be trivial proxies. @@ -186,4 +179,4 @@ export class DedicatedCryptoWorker { } } -Comlink.expose(DedicatedCryptoWorker, self); +expose(DedicatedCryptoWorker); diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/shared/crypto/internal/libsodium.ts index 9e2df77dfe..7a3e720319 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/shared/crypto/internal/libsodium.ts @@ -4,7 +4,9 @@ * * All functions are stateless, async, and safe to use in Web Workers. * - * Docs for the JS library: https://github.com/jedisct1/libsodium.js + * Docs for the JS library: https://github.com/jedisct1/libsodium.js. + * + * To see where this code fits, see [Note: Crypto code hierarchy]. */ import { mergeUint8Arrays } from "@/utils/array"; import { CustomError } from "@ente/shared/error"; From 85020a490eb31782563548f1e08833050ef4b094 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 12:16:59 +0530 Subject: [PATCH 144/211] Rearrange --- web/packages/base/crypto/ente.ts | 30 +++++++++---------- .../internal => base/crypto}/libsodium.ts | 4 +++ .../crypto => base/crypto/worker}/index.ts | 6 ++-- .../crypto/worker/worker.ts} | 4 +-- 4 files changed, 24 insertions(+), 20 deletions(-) rename web/packages/{shared/crypto/internal => base/crypto}/libsodium.ts (98%) rename web/packages/{shared/crypto => base/crypto/worker}/index.ts (80%) rename web/packages/{shared/crypto/internal/crypto.worker.ts => base/crypto/worker/worker.ts} (97%) diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index e28583137f..7b09d3c02f 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -3,25 +3,25 @@ * * [Note: Crypto code hierarchy] * - * 1. ente.ts or crypto.worker.ts (high level, Ente specific). - * 2. internal/libsodium.ts (wrappers over libsodium) - * 3. libsodium (JS bindings). + * 1. crypto/ente.ts (Ente specific higher level functions) + * 2. crypto/libsodium.ts (More primitive wrappers over libsodium) + * 3. libsodium-wrappers (JavaScript bindings to libsodium) * * Our cryptography primitives are provided by libsodium, specifically, its * JavaScript bindings ("libsodium-wrappers"). This is the lowest layer. * - * Direct usage of "libsodium-wrappers" is restricted to - * `crypto/internal/libsodium.ts`. This is the next higher layer, and the first - * one that our code should directly use. Usually the functions in this file are - * thin wrappers over the raw libsodium APIs, with a bit of massaging or - * book-keeping. They also ensure that sodium.ready has been called before - * accessing libsodium's APIs, thus all the functions it exposes are async. + * Direct usage of "libsodium-wrappers" is restricted to `crypto/libsodium.ts`. + * This is the next higher layer, and the first one that our code should + * directly use. Usually the functions in this file are thin wrappers over the + * raw libsodium APIs, with a bit of massaging. They also ensure that + * sodium.ready has been called before accessing libsodium's APIs, thus all the + * functions it exposes are async. * * The final layer is this file, `crypto/ente.ts`. These are usually thin - * wrappers themselves over functions exposed by `internal/libsodium.ts`, but - * the difference is that the functions in ente.ts don't talk in terms of the - * crypto algorithms, but rather in terms the higher-level Ente specific goal we - * are trying to accomplish. + * wrappers themselves over functions exposed by `crypto/libsodium.ts`, but the + * difference is that the functions in ente.ts don't talk in terms of the crypto + * algorithms, but rather in terms the higher-level Ente specific goal we are + * trying to accomplish. * * There is an additional actor in the play. Cryptographic operations are CPU * intensive and would cause the UI to stutter if used directly on the main @@ -42,9 +42,9 @@ * recommendation though (in circumstances where we create more crypto workers * instead of using the shared one). */ -import ComlinkCryptoWorker from "@ente/shared/crypto"; -import * as libsodium from "@ente/shared/crypto/internal/libsodium"; import { inWorker } from "../env"; +import * as libsodium from "./libsodium"; +import ComlinkCryptoWorker from "./worker"; const cryptoWorker = () => ComlinkCryptoWorker.getInstance(); diff --git a/web/packages/shared/crypto/internal/libsodium.ts b/web/packages/base/crypto/libsodium.ts similarity index 98% rename from web/packages/shared/crypto/internal/libsodium.ts rename to web/packages/base/crypto/libsodium.ts index 7a3e720319..cf30256a86 100644 --- a/web/packages/shared/crypto/internal/libsodium.ts +++ b/web/packages/base/crypto/libsodium.ts @@ -311,6 +311,8 @@ export const decryptChaCha = async ( pullState, buffer, ); + // TODO: + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (!pullResult.message) { throw new Error(CustomError.PROCESSING_FAILED); } @@ -343,6 +345,8 @@ export async function decryptFileChunk( pullState, data, ); + // TODO: + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (!pullResult.message) { throw new Error(CustomError.PROCESSING_FAILED); } diff --git a/web/packages/shared/crypto/index.ts b/web/packages/base/crypto/worker/index.ts similarity index 80% rename from web/packages/shared/crypto/index.ts rename to web/packages/base/crypto/worker/index.ts index ed3d47e155..8d4f50e4bf 100644 --- a/web/packages/shared/crypto/index.ts +++ b/web/packages/base/crypto/worker/index.ts @@ -1,6 +1,6 @@ import { ComlinkWorker } from "@/base/worker/comlink-worker"; import type { Remote } from "comlink"; -import { type DedicatedCryptoWorker } from "./internal/crypto.worker"; +import { type DedicatedCryptoWorker } from "./worker"; class ComlinkCryptoWorker { private comlinkWorkerInstance: @@ -18,8 +18,8 @@ class ComlinkCryptoWorker { export const getDedicatedCryptoWorker = () => { const cryptoComlinkWorker = new ComlinkWorker( - "ente-crypto-worker", - new Worker(new URL("internal/crypto.worker.ts", import.meta.url)), + "Crypto", + new Worker(new URL("./worker.ts", import.meta.url)), ); return cryptoComlinkWorker; }; diff --git a/web/packages/shared/crypto/internal/crypto.worker.ts b/web/packages/base/crypto/worker/worker.ts similarity index 97% rename from web/packages/shared/crypto/internal/crypto.worker.ts rename to web/packages/base/crypto/worker/worker.ts index 319d75c57d..9fff54f61f 100644 --- a/web/packages/shared/crypto/internal/crypto.worker.ts +++ b/web/packages/base/crypto/worker/worker.ts @@ -1,7 +1,7 @@ -import * as ente from "@/base/crypto/ente"; -import * as libsodium from "@ente/shared/crypto/internal/libsodium"; import { expose } from "comlink"; import type { StateAddress } from "libsodium-wrappers"; +import * as ente from "../ente"; +import * as libsodium from "../libsodium"; /** * A web worker that exposes some of the functions defined in either the Ente From 231e831c75122f229f1ad08a42ef1419ebaaa0b5 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 12:24:38 +0530 Subject: [PATCH 145/211] Refactor --- web/packages/base/crypto/ente.ts | 8 +++--- web/packages/base/crypto/worker/index.ts | 32 ++++++++--------------- web/packages/base/crypto/worker/worker.ts | 9 ++++--- 3 files changed, 19 insertions(+), 30 deletions(-) diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index 7b09d3c02f..74737db1db 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -18,7 +18,7 @@ * functions it exposes are async. * * The final layer is this file, `crypto/ente.ts`. These are usually thin - * wrappers themselves over functions exposed by `crypto/libsodium.ts`, but the + * compositions of functionality exposed by `crypto/libsodium.ts`, but the * difference is that the functions in ente.ts don't talk in terms of the crypto * algorithms, but rather in terms the higher-level Ente specific goal we are * trying to accomplish. @@ -44,9 +44,7 @@ */ import { inWorker } from "../env"; import * as libsodium from "./libsodium"; -import ComlinkCryptoWorker from "./worker"; - -const cryptoWorker = () => ComlinkCryptoWorker.getInstance(); +import { sharedCryptoWorker } from "./worker"; /** * Encrypt arbitrary data associated with an Ente object (file, collection, @@ -236,7 +234,7 @@ export const decryptMetadataBytes = ( ) => inWorker() ? decryptMetadataBytesI(encryptedDataB64, decryptionHeaderB64, keyB64) - : cryptoWorker().then((cw) => + : sharedCryptoWorker().then((cw) => cw.decryptMetadataBytes( encryptedDataB64, decryptionHeaderB64, diff --git a/web/packages/base/crypto/worker/index.ts b/web/packages/base/crypto/worker/index.ts index 8d4f50e4bf..040250c286 100644 --- a/web/packages/base/crypto/worker/index.ts +++ b/web/packages/base/crypto/worker/index.ts @@ -1,27 +1,17 @@ import { ComlinkWorker } from "@/base/worker/comlink-worker"; -import type { Remote } from "comlink"; -import { type DedicatedCryptoWorker } from "./worker"; +import type { CryptoWorker } from "./worker"; -class ComlinkCryptoWorker { - private comlinkWorkerInstance: - | Promise> - | undefined; +/** Cached instance of the {@link ComlinkWorker} that wraps our web worker. */ +let _comlinkWorker: ComlinkWorker | undefined; - async getInstance() { - if (!this.comlinkWorkerInstance) { - const comlinkWorker = getDedicatedCryptoWorker(); - this.comlinkWorkerInstance = comlinkWorker.remote; - } - return this.comlinkWorkerInstance; - } -} +/** + * Lazily created, cached, instance of a CryptoWorker web worker. + */ +export const sharedCryptoWorker = async () => + (_comlinkWorker ??= createComlinkWorker()).remote; -export const getDedicatedCryptoWorker = () => { - const cryptoComlinkWorker = new ComlinkWorker( +const createComlinkWorker = () => + new ComlinkWorker( "Crypto", - new Worker(new URL("./worker.ts", import.meta.url)), + new Worker(new URL("worker.ts", import.meta.url)), ); - return cryptoComlinkWorker; -}; - -export default new ComlinkCryptoWorker(); diff --git a/web/packages/base/crypto/worker/worker.ts b/web/packages/base/crypto/worker/worker.ts index 9fff54f61f..298c7150d0 100644 --- a/web/packages/base/crypto/worker/worker.ts +++ b/web/packages/base/crypto/worker/worker.ts @@ -5,14 +5,15 @@ import * as libsodium from "../libsodium"; /** * A web worker that exposes some of the functions defined in either the Ente - * specific layer (base/crypto/ente.ts) or the internal libsodium layer - * (internal/libsodium.ts). + * specific layer (crypto/ente.ts) or the libsodium layer (crypto/libsodium.ts). * * See: [Note: Crypto code hierarchy]. * * Note: Keep these methods logic free. They are meant to be trivial proxies. */ -export class DedicatedCryptoWorker { +export class CryptoWorker { + // TODO: -- AUDIT BELOW -- + async decryptThumbnail( encryptedData: Uint8Array, headerB64: string, @@ -179,4 +180,4 @@ export class DedicatedCryptoWorker { } } -expose(DedicatedCryptoWorker); +expose(CryptoWorker); From f5b6145da19edbadd2730228fcdd23f2e85f6840 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 12:45:45 +0530 Subject: [PATCH 146/211] Impl 1 --- web/packages/base/crypto/ente-impl.ts | 74 +++++++++++++++++++++++ web/packages/base/crypto/ente.ts | 50 ++++++++------- web/packages/base/crypto/worker/worker.ts | 42 +++++-------- 3 files changed, 113 insertions(+), 53 deletions(-) create mode 100644 web/packages/base/crypto/ente-impl.ts diff --git a/web/packages/base/crypto/ente-impl.ts b/web/packages/base/crypto/ente-impl.ts new file mode 100644 index 0000000000..e8580ae008 --- /dev/null +++ b/web/packages/base/crypto/ente-impl.ts @@ -0,0 +1,74 @@ +/** Careful when adding add other imports! */ +import * as libsodium from "./libsodium"; + +export const encryptAssociatedDataI = libsodium.encryptChaChaOneShot; + +export const encryptThumbnailI = encryptAssociatedDataI; + +export const encryptFileEmbeddingI = async ( + data: Uint8Array, + keyB64: string, +) => { + const { encryptedData, decryptionHeaderB64 } = await encryptAssociatedDataI( + data, + keyB64, + ); + return { + encryptedDataB64: await libsodium.toB64(encryptedData), + decryptionHeaderB64, + }; +}; + +export const encryptMetadataI = async (metadata: unknown, keyB64: string) => { + const encodedMetadata = new TextEncoder().encode(JSON.stringify(metadata)); + + const { encryptedData, decryptionHeaderB64 } = await encryptAssociatedDataI( + encodedMetadata, + keyB64, + ); + return { + encryptedDataB64: await libsodium.toB64(encryptedData), + decryptionHeaderB64, + }; +}; + +export const decryptAssociatedDataI = libsodium.decryptChaChaOneShot; + +export const decryptThumbnailI = decryptAssociatedDataI; + +export const decryptFileEmbeddingI = async ( + encryptedDataB64: string, + decryptionHeaderB64: string, + keyB64: string, +) => + decryptAssociatedDataI( + await libsodium.fromB64(encryptedDataB64), + decryptionHeaderB64, + keyB64, + ); + +export const decryptMetadataI = async ( + encryptedDataB64: string, + decryptionHeaderB64: string, + keyB64: string, +) => + JSON.parse( + new TextDecoder().decode( + await decryptMetadataBytesI( + encryptedDataB64, + decryptionHeaderB64, + keyB64, + ), + ), + ) as unknown; + +export const decryptMetadataBytesI = async ( + encryptedDataB64: string, + decryptionHeaderB64: string, + keyB64: string, +) => + await decryptAssociatedDataI( + await libsodium.fromB64(encryptedDataB64), + decryptionHeaderB64, + keyB64, + ); diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index 74737db1db..c5ebca7eab 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -36,13 +36,20 @@ * function in a shared "crypto" web worker (which then invokes the * implementation, but this time in the context of a web worker). * + * To avoid a circular dependency during webpack imports, we need to keep the + * implementation functions in a separate file (ente-impl.ts). This is a bit + * unfortunate, since it makes them harder to read and reason about (since their + * documentation and parameter names are all in ente.ts). + * * Some older code directly calls the functions in the shared crypto.worker.ts, * but that should be avoided since it makes the code not behave the way we want * when we're already in a web worker. There are exceptions to this * recommendation though (in circumstances where we create more crypto workers * instead of using the shared one). */ +import { assertionFailed } from "../assert"; import { inWorker } from "../env"; +import * as ei from "./ente-impl"; import * as libsodium from "./libsodium"; import { sharedCryptoWorker } from "./worker"; @@ -182,12 +189,14 @@ export const decryptFileEmbedding = async ( encryptedDataB64: string, decryptionHeaderB64: string, keyB64: string, -) => - decryptAssociatedData( - await libsodium.fromB64(encryptedDataB64), +) => { + if (!inWorker()) assertionFailed("Only implemented for use in web workers"); + return ei.decryptFileEmbeddingI( + encryptedDataB64, decryptionHeaderB64, keyB64, ); +}; /** * Decrypt the metadata associated with an Ente object (file, collection or @@ -212,15 +221,11 @@ export const decryptMetadata = async ( decryptionHeaderB64: string, keyB64: string, ) => - JSON.parse( - new TextDecoder().decode( - await decryptMetadataBytes( - encryptedDataB64, - decryptionHeaderB64, - keyB64, - ), - ), - ) as unknown; + inWorker() + ? ei.decryptMetadataI(encryptedDataB64, decryptionHeaderB64, keyB64) + : sharedCryptoWorker().then((w) => + w.decryptMetadata(encryptedDataB64, decryptionHeaderB64, keyB64), + ); /** * A variant of {@link decryptMetadata} that does not attempt to parse the @@ -233,22 +238,15 @@ export const decryptMetadataBytes = ( keyB64: string, ) => inWorker() - ? decryptMetadataBytesI(encryptedDataB64, decryptionHeaderB64, keyB64) - : sharedCryptoWorker().then((cw) => - cw.decryptMetadataBytes( + ? ei.decryptMetadataBytesI( + encryptedDataB64, + decryptionHeaderB64, + keyB64, + ) + : sharedCryptoWorker().then((w) => + w.decryptMetadataBytes( encryptedDataB64, decryptionHeaderB64, keyB64, ), ); - -export const decryptMetadataBytesI = async ( - encryptedDataB64: string, - decryptionHeaderB64: string, - keyB64: string, -) => - await decryptAssociatedData( - await libsodium.fromB64(encryptedDataB64), - decryptionHeaderB64, - keyB64, - ); diff --git a/web/packages/base/crypto/worker/worker.ts b/web/packages/base/crypto/worker/worker.ts index 298c7150d0..ab775b6a61 100644 --- a/web/packages/base/crypto/worker/worker.ts +++ b/web/packages/base/crypto/worker/worker.ts @@ -1,6 +1,6 @@ import { expose } from "comlink"; import type { StateAddress } from "libsodium-wrappers"; -import * as ente from "../ente"; +import * as ei from "../ente-impl"; import * as libsodium from "../libsodium"; /** @@ -12,42 +12,30 @@ import * as libsodium from "../libsodium"; * Note: Keep these methods logic free. They are meant to be trivial proxies. */ export class CryptoWorker { - // TODO: -- AUDIT BELOW -- - - async decryptThumbnail( - encryptedData: Uint8Array, - headerB64: string, - keyB64: string, - ) { - return ente.decryptThumbnail(encryptedData, headerB64, keyB64); + async encryptThumbnail(a: Uint8Array, b: string) { + return ei.encryptThumbnailI(a, b); } - async decryptMetadata( - encryptedDataB64: string, - decryptionHeaderB64: string, - keyB64: string, - ) { - return ente.decryptMetadata( - encryptedDataB64, - decryptionHeaderB64, - keyB64, - ); + async encryptMetadata(a: unknown, b: string) { + return ei.encryptMetadataI(a, b); } - async decryptMetadataBytes(a: string, b: string, c: string) { - return ente.decryptMetadataBytesI(a, b, c); + async decryptThumbnail(a: Uint8Array, b: string, c: string) { + return ei.decryptThumbnailI(a, b, c); } - async decryptFile(fileData: Uint8Array, header: Uint8Array, key: string) { - return libsodium.decryptChaCha(fileData, header, key); + async decryptMetadata(a: string, b: string, c: string) { + return ei.decryptMetadataI(a, b, c); } - async encryptThumbnail(data: Uint8Array, keyB64: string) { - return ente.encryptThumbnail(data, keyB64); + async decryptMetadataBytes(a: string, b: string, c: string) { + return ei.decryptMetadataBytesI(a, b, c); } - async encryptMetadata(metadata: unknown, keyB64: string) { - return ente.encryptMetadata(metadata, keyB64); + // TODO: -- AUDIT BELOW -- + + async decryptFile(fileData: Uint8Array, header: Uint8Array, key: string) { + return libsodium.decryptChaCha(fileData, header, key); } async encryptFile(fileData: Uint8Array) { From ca9726969c5e69547deadf912aad2cbebce7dce8 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 13:03:37 +0530 Subject: [PATCH 147/211] Impl 2 --- web/packages/base/crypto/ente.ts | 60 +++++++++++++------------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index c5ebca7eab..98d768ba39 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -53,6 +53,16 @@ import * as ei from "./ente-impl"; import * as libsodium from "./libsodium"; import { sharedCryptoWorker } from "./worker"; +/** + * Some functions we haven't yet needed to use on the main thread. These don't + * have a corresponding sharedCryptoWorker interface. This assertion will let us + * know when we need to implement them (in production it'll just log a warning). + */ +const assertInWorker = (x: T): T => { + if (!inWorker()) assertionFailed("Currently only usable in a web worker"); + return x; +}; + /** * Encrypt arbitrary data associated with an Ente object (file, collection, * entity) using the object's key. @@ -69,7 +79,8 @@ import { sharedCryptoWorker } from "./worker"; * * @returns The encrypted data and the (base64 encoded) decryption header. */ -export const encryptAssociatedData = libsodium.encryptChaChaOneShot; +export const encryptAssociatedData = (data: Uint8Array, keyB64: string) => + assertInWorker(ei.encryptAssociatedDataI(data, keyB64)); /** * Encrypt the thumbnail for a file. @@ -83,7 +94,8 @@ export const encryptAssociatedData = libsodium.encryptChaChaOneShot; * @returns The encrypted thumbnail, and the associated decryption header * (base64 encoded). */ -export const encryptThumbnail = encryptAssociatedData; +export const encryptThumbnail = (data: Uint8Array, keyB64: string) => + assertInWorker(ei.encryptThumbnailI(data, keyB64)); /** * Encrypted the embedding associated with a file using the file's key. @@ -95,19 +107,8 @@ export const encryptThumbnail = encryptAssociatedData; * * Use {@link decryptFileEmbedding} to decrypt the result. */ -export const encryptFileEmbedding = async ( - data: Uint8Array, - keyB64: string, -) => { - const { encryptedData, decryptionHeaderB64 } = await encryptAssociatedData( - data, - keyB64, - ); - return { - encryptedDataB64: await libsodium.toB64(encryptedData), - decryptionHeaderB64, - }; -}; +export const encryptFileEmbedding = async (data: Uint8Array, keyB64: string) => + assertInWorker(ei.encryptFileEmbeddingI(data, keyB64)); /** * Encrypt the metadata associated with an Ente object (file, collection or @@ -130,18 +131,8 @@ export const encryptFileEmbedding = async ( * * @returns The encrypted data and decryption header, both as base64 strings. */ -export const encryptMetadata = async (metadata: unknown, keyB64: string) => { - const encodedMetadata = new TextEncoder().encode(JSON.stringify(metadata)); - - const { encryptedData, decryptionHeaderB64 } = await encryptAssociatedData( - encodedMetadata, - keyB64, - ); - return { - encryptedDataB64: await libsodium.toB64(encryptedData), - decryptionHeaderB64, - }; -}; +export const encryptMetadata = async (metadata: unknown, keyB64: string) => + assertInWorker(ei.encryptMetadataI(metadata, keyB64)); /** * Decrypt arbitrary data associated with an Ente object (file, collection or @@ -161,7 +152,10 @@ export const encryptMetadata = async (metadata: unknown, keyB64: string) => { * * @returns The decrypted bytes. */ -export const decryptAssociatedData = libsodium.decryptChaChaOneShot; +export const decryptAssociatedData = ( encryptedData: Uint8Array, + headerB64: string, + keyB64: string, +) => libsodium.decryptChaChaOneShot; /** * Decrypt the thumbnail for a file. @@ -189,14 +183,10 @@ export const decryptFileEmbedding = async ( encryptedDataB64: string, decryptionHeaderB64: string, keyB64: string, -) => { - if (!inWorker()) assertionFailed("Only implemented for use in web workers"); - return ei.decryptFileEmbeddingI( - encryptedDataB64, - decryptionHeaderB64, - keyB64, +) => + assertInWorker( + ei.decryptFileEmbeddingI(encryptedDataB64, decryptionHeaderB64, keyB64), ); -}; /** * Decrypt the metadata associated with an Ente object (file, collection or From d5d7786b24cde5b62aa11d8f778911a85252ad7d Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 17:09:43 +0530 Subject: [PATCH 148/211] Types --- web/packages/base/crypto/types.ts | 74 +++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 web/packages/base/crypto/types.ts diff --git a/web/packages/base/crypto/types.ts b/web/packages/base/crypto/types.ts new file mode 100644 index 0000000000..1411259c4d --- /dev/null +++ b/web/packages/base/crypto/types.ts @@ -0,0 +1,74 @@ +/** + * An encryption request with the plaintext data as bytes. + */ +export interface EncryptBytes { + /** + * A {@link Uint8Array} containing the bytes to encrypt. + */ + data: Uint8Array; + /** + * A base64 string containing the encryption key. + */ + keyB64: string; +} + +/** + * An encryption request with the plaintext data as a JSON value. + */ +export interface EncryptJSON { + /** + * The JSON value to encrypt. + * + * This can be an arbitrary JSON value, but since TypeScript currently + * doesn't have a native JSON type, it is typed as {@link unknown}. + */ + jsonValue: unknown; + /** + * A base64 string containing the encryption key. + */ + keyB64: string; +} + +/** + * A decryption request with the encrypted data as a base64 encoded string. + */ +export interface DecryptB64 { + /** + * A base64 string containing the data to decrypt. + */ + encryptedDataB64: string; + /** + * A base64 string containing the decryption header that was produced during + * encryption. + * + * The header contains a random nonce and other libsodium metadata. It does + * not need to be kept secret. + */ + decryptionHeaderB64: string; + /** + * A base64 string containing the encryption key. + */ + keyB64: string; +} + +/** + * A decryption request with the encrypted data as bytes. + */ +export interface DecryptBytes { + /** + * A {@link Uint8Array} containing the bytes to decrypt. + */ + encryptedData: Uint8Array; + /** + * A base64 string containing the decryption header that was produced during + * encryption. + * + * The header contains a random nonce and other libsodium metadata. It does + * not need to be kept secret. + */ + decryptionHeaderB64: string; + /** + * A base64 string containing the encryption key. + */ + keyB64: string; +} From 97c92531274e5b5a174849ebb1c06d4aa2f7bc68 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 10 Aug 2024 17:16:35 +0530 Subject: [PATCH 149/211] [server] Support for reporting preview video --- server/ente/filedata/putfiledata.go | 10 ++++++++++ server/pkg/controller/filedata/controller.go | 17 +++++++++++++++-- server/pkg/controller/filedata/s3.go | 20 ++++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go index 0d70570da3..08ef42ef1b 100644 --- a/server/ente/filedata/putfiledata.go +++ b/server/ente/filedata/putfiledata.go @@ -54,3 +54,13 @@ func (r PutFileDataRequest) S3FileMetadataObjectKey(ownerID int64) string { } panic(fmt.Sprintf("S3FileMetadata should not be written for %s type", r.Type)) } + +func (r PutFileDataRequest) S3FileObjectKey(ownerID int64) string { + if r.Type == ente.PreviewVideo { + return previewVideoPath(r.FileID, ownerID) + } + if r.Type == ente.PreviewImage { + return previewImagePath(r.FileID, ownerID) + } + panic(fmt.Sprintf("S3FileObjectKey should not be written for %s type", r.Type)) +} diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index b01cec9c48..9809d7f217 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -19,6 +19,7 @@ import ( "github.com/ente-io/stacktrace" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" + "strings" "sync" gTime "time" ) @@ -87,10 +88,21 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR if err != nil { return stacktrace.Propagate(err, "") } - if req.Type != ente.DerivedMeta { + if req.Type != ente.DerivedMeta && req.Type != ente.PreviewVideo { return stacktrace.Propagate(ente.NewBadRequestWithMessage("unsupported object type "+string(req.Type)), "") } fileOwnerID := userID + bucketID := c.S3Config.GetBucketID(req.Type) + if req.Type == ente.PreviewVideo { + fileObjectKey := req.S3FileObjectKey(fileOwnerID) + if !strings.Contains(*req.ObjectKey, fileObjectKey) { + return stacktrace.Propagate(ente.NewBadRequestWithMessage("objectKey should contain the file object key"), "") + } + err = c.copyObject(*req.ObjectKey, fileObjectKey, bucketID) + if err != nil { + return err + } + } objectKey := req.S3FileMetadataObjectKey(fileOwnerID) obj := fileData.S3FileMetadata{ Version: *req.Version, @@ -98,12 +110,13 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR DecryptionHeader: *req.DecryptionHeader, Client: network.GetClientInfo(ctx), } - bucketID := c.S3Config.GetBucketID(req.Type) + size, uploadErr := c.uploadObject(obj, objectKey, bucketID) if uploadErr != nil { log.Error(uploadErr) return stacktrace.Propagate(uploadErr, "") } + row := fileData.Row{ FileID: req.FileID, Type: req.Type, diff --git a/server/pkg/controller/filedata/s3.go b/server/pkg/controller/filedata/s3.go index 415e28cb28..e6f1200bf8 100644 --- a/server/pkg/controller/filedata/s3.go +++ b/server/pkg/controller/filedata/s3.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3/s3manager" @@ -90,3 +91,22 @@ func (c *Controller) uploadObject(obj fileData.S3FileMetadata, objectKey string, log.Infof("Uploaded to bucket %s", result.Location) return int64(len(embeddingObj)), nil } + +// copyObject copies the object from srcObjectKey to destObjectKey in the same bucket and returns the object size +func (c *Controller) copyObject(srcObjectKey string, destObjectKey string, bucketID string) error { + bucket := c.S3Config.GetBucket(bucketID) + s3Client := c.S3Config.GetS3Client(bucketID) + copySource := fmt.Sprintf("%s/%s", *bucket, srcObjectKey) + copyInput := &s3.CopyObjectInput{ + Bucket: bucket, + CopySource: ©Source, + Key: aws.String(destObjectKey), + } + + _, err := s3Client.CopyObject(copyInput) + if err != nil { + return fmt.Errorf("failed to copy (%s) from %s to %s: %v", bucketID, srcObjectKey, destObjectKey, err) + } + log.Infof("Copied (%s) from %s to %s", bucketID, srcObjectKey, destObjectKey) + return nil +} From bac660f7a05cef48e3164c28efc1c60fe04d4dc8 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Sat, 10 Aug 2024 17:20:03 +0530 Subject: [PATCH 150/211] [server] Lint fix --- server/pkg/controller/filedata/controller.go | 8 ++------ server/pkg/controller/filedata/delete.go | 9 ++++----- server/pkg/repo/filedata/repository.go | 2 +- server/pkg/utils/s3config/s3config.go | 2 +- 4 files changed, 8 insertions(+), 13 deletions(-) diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index 9809d7f217..dbdb418fa4 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -24,10 +24,6 @@ import ( gTime "time" ) -const ( - embeddingFetchTimeout = 10 * gTime.Second -) - // _fetchConfig is the configuration for the fetching objects from S3 type _fetchConfig struct { RetryCount int @@ -173,7 +169,7 @@ func (c *Controller) GetFilesData(ctx *gin.Context, req fileData.GetFilesData) ( errFileIds := make([]int64, 0) for i := range doRows { dbFileIds = append(dbFileIds, doRows[i].FileID) - if doRows[i].IsDeleted == false { + if !doRows[i].IsDeleted { activeRows = append(activeRows, doRows[i]) } } @@ -209,7 +205,7 @@ func (c *Controller) GetFilesData(ctx *gin.Context, req fileData.GetFilesData) ( func (c *Controller) getS3FileMetadataParallel(dbRows []fileData.Row) ([]bulkS3MetaFetchResult, error) { var wg sync.WaitGroup embeddingObjects := make([]bulkS3MetaFetchResult, len(dbRows)) - for i, _ := range dbRows { + for i := range dbRows { dbRow := dbRows[i] wg.Add(1) globalFileFetchSemaphore <- struct{}{} // Acquire from global semaphore diff --git a/server/pkg/controller/filedata/delete.go b/server/pkg/controller/filedata/delete.go index 782f30d370..a610282095 100644 --- a/server/pkg/controller/filedata/delete.go +++ b/server/pkg/controller/filedata/delete.go @@ -8,7 +8,7 @@ import ( "github.com/ente-io/museum/ente/filedata" fileDataRepo "github.com/ente-io/museum/pkg/repo/filedata" enteTime "github.com/ente-io/museum/pkg/utils/time" - "github.com/sirupsen/logrus" + log "github.com/sirupsen/logrus" "time" ) @@ -80,8 +80,7 @@ func (c *Controller) deleteFileRow(fileDataRow filedata.Row) error { } ctxLogger := log.WithField("file_id", fileDataRow.DeleteFromBuckets).WithField("type", fileDataRow.Type).WithField("user_id", fileDataRow.UserID) objectKeys := filedata.AllObjects(fileID, ownerID, fileDataRow.Type) - bucketColumnMap := make(map[string]string) - bucketColumnMap, err = getMapOfBucketItToColumn(fileDataRow) + bucketColumnMap, err := getMapOfBucketItToColumn(fileDataRow) if err != nil { ctxLogger.WithError(err).Error("Failed to get bucketColumnMap") return err @@ -91,7 +90,7 @@ func (c *Controller) deleteFileRow(fileDataRow filedata.Row) error { for _, objectKey := range objectKeys { err := c.ObjectCleanupController.DeleteObjectFromDataCenter(objectKey, bucketID) if err != nil { - ctxLogger.WithError(err).WithFields(logrus.Fields{ + ctxLogger.WithError(err).WithFields(log.Fields{ "bucketID": bucketID, "column": columnName, "objectKey": objectKey, @@ -101,7 +100,7 @@ func (c *Controller) deleteFileRow(fileDataRow filedata.Row) error { } dbErr := c.Repo.RemoveBucket(fileDataRow, bucketID, columnName) if dbErr != nil { - ctxLogger.WithError(dbErr).WithFields(logrus.Fields{ + ctxLogger.WithError(dbErr).WithFields(log.Fields{ "bucketID": bucketID, "column": columnName, }).Error("Failed to remove bucket from db") diff --git a/server/pkg/repo/filedata/repository.go b/server/pkg/repo/filedata/repository.go index 0293b05f0f..9f5dc5cf3f 100644 --- a/server/pkg/repo/filedata/repository.go +++ b/server/pkg/repo/filedata/repository.go @@ -209,7 +209,7 @@ func (r *Repository) RegisterReplicationAttempt(ctx context.Context, row filedat if array.StringInList(dstBucketID, row.DeleteFromBuckets) { return r.MoveBetweenBuckets(row, dstBucketID, DeletionColumn, InflightRepColumn) } - if array.StringInList(dstBucketID, row.InflightReplicas) == false { + if !array.StringInList(dstBucketID, row.InflightReplicas) { return r.AddBucket(row, dstBucketID, InflightRepColumn) } return nil diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index b027ffac35..46ece00ef0 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -155,7 +155,7 @@ func (config *S3Config) initialize() { } if err := viper.Sub("s3").Unmarshal(&config.fileDataConfig); err != nil { - log.Fatal("Unable to decode into struct: %v\n", err) + log.Fatalf("Unable to decode into struct: %v\n", err) return } From 4e49a352be0c46c270748d5ae3d99866c23225f3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 17:39:29 +0530 Subject: [PATCH 151/211] Integrate --- web/packages/base/crypto/ente-impl.ts | 27 ++++++++---------- web/packages/base/crypto/ente.ts | 40 ++++++++++----------------- web/packages/base/crypto/libsodium.ts | 37 +++++++++---------------- web/packages/base/crypto/types.ts | 39 ++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 65 deletions(-) diff --git a/web/packages/base/crypto/ente-impl.ts b/web/packages/base/crypto/ente-impl.ts index e8580ae008..7d23f01f60 100644 --- a/web/packages/base/crypto/ente-impl.ts +++ b/web/packages/base/crypto/ente-impl.ts @@ -1,30 +1,25 @@ /** Careful when adding add other imports! */ import * as libsodium from "./libsodium"; +import type { EncryptBytes, EncryptJSON } from "./types"; -export const encryptAssociatedDataI = libsodium.encryptChaChaOneShot; +export const _encryptAssociatedData = libsodium.encryptChaChaOneShot; -export const encryptThumbnailI = encryptAssociatedDataI; +export const _encryptThumbnail = _encryptAssociatedData; -export const encryptFileEmbeddingI = async ( - data: Uint8Array, - keyB64: string, -) => { - const { encryptedData, decryptionHeaderB64 } = await encryptAssociatedDataI( - data, - keyB64, - ); +export const _encryptFileEmbedding = async (r: EncryptBytes) => { + const { encryptedData, decryptionHeaderB64 } = + await _encryptAssociatedData(r); return { encryptedDataB64: await libsodium.toB64(encryptedData), decryptionHeaderB64, }; }; -export const encryptMetadataI = async (metadata: unknown, keyB64: string) => { - const encodedMetadata = new TextEncoder().encode(JSON.stringify(metadata)); +export const _encryptMetadata = async ({ jsonValue, keyB64 }: EncryptJSON) => { + const data = new TextEncoder().encode(JSON.stringify(jsonValue)); - const { encryptedData, decryptionHeaderB64 } = await encryptAssociatedDataI( - encodedMetadata, - keyB64, + const { encryptedData, decryptionHeaderB64 } = await _encryptAssociatedData( + { data, keyB64 }, ); return { encryptedDataB64: await libsodium.toB64(encryptedData), @@ -32,7 +27,7 @@ export const encryptMetadataI = async (metadata: unknown, keyB64: string) => { }; }; -export const decryptAssociatedDataI = libsodium.decryptChaChaOneShot; +export const _decryptAssociatedData = libsodium.decryptChaChaOneShot; export const decryptThumbnailI = decryptAssociatedDataI; diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index 98d768ba39..7897288cbb 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -17,7 +17,7 @@ * sodium.ready has been called before accessing libsodium's APIs, thus all the * functions it exposes are async. * - * The final layer is this file, `crypto/ente.ts`. These are usually thin + * The highest layer is this file, `crypto/ente.ts`. These are usually simple * compositions of functionality exposed by `crypto/libsodium.ts`, but the * difference is that the functions in ente.ts don't talk in terms of the crypto * algorithms, but rather in terms the higher-level Ente specific goal we are @@ -30,7 +30,7 @@ * to another worker is wasteful. * * To handle both these scenario, each function in this file is split into the - * external API, and the underlying implementation (denoted by an "I" suffix). + * external API, and the underlying implementation (denoted by an "_" prefix). * The external API functions check to see if we're already in a web worker, and * if so directly invoke the implementation. Otherwise the call the sibling * function in a shared "crypto" web worker (which then invokes the @@ -51,12 +51,16 @@ import { assertionFailed } from "../assert"; import { inWorker } from "../env"; import * as ei from "./ente-impl"; import * as libsodium from "./libsodium"; +import type { EncryptBytes } from "./types"; import { sharedCryptoWorker } from "./worker"; /** - * Some functions we haven't yet needed to use on the main thread. These don't - * have a corresponding sharedCryptoWorker interface. This assertion will let us - * know when we need to implement them (in production it'll just log a warning). + * Some of these functions have not yet been needed on the main thread, and for + * these we don't have a corresponding sharedCryptoWorker method. + * + * This assertion will let us know when we need to implement them (this'll + * gracefully degrade in production: the functionality will work, just that the + * crypto will happen on the main thread itself). */ const assertInWorker = (x: T): T => { if (!inWorker()) assertionFailed("Currently only usable in a web worker"); @@ -70,32 +74,17 @@ const assertInWorker = (x: T): T => { * Use {@link decryptAssociatedData} to decrypt the result. * * See {@link encryptChaChaOneShot} for the implementation details. - * - * @param data A {@link Uint8Array} containing the bytes to encrypt. - * - * @param keyB64 base64 string containing the encryption key. This is expected - * to the key of the object with which {@link data} is associated. For example, - * if this is data associated with a file, then this will be the file's key. - * - * @returns The encrypted data and the (base64 encoded) decryption header. */ -export const encryptAssociatedData = (data: Uint8Array, keyB64: string) => - assertInWorker(ei.encryptAssociatedDataI(data, keyB64)); +export const encryptAssociatedData = (r: EncryptBytes) => + assertInWorker(ei._encryptAssociatedData(r)); /** * Encrypt the thumbnail for a file. * * This is just an alias for {@link encryptAssociatedData}. - * - * @param data The thumbnail's data. - * - * @param keyB64 The key associated with the file whose thumbnail this is. - * - * @returns The encrypted thumbnail, and the associated decryption header - * (base64 encoded). */ -export const encryptThumbnail = (data: Uint8Array, keyB64: string) => - assertInWorker(ei.encryptThumbnailI(data, keyB64)); +export const encryptThumbnail = (r: EncryptBytes) => + assertInWorker(ei._encryptThumbnail(r)); /** * Encrypted the embedding associated with a file using the file's key. @@ -152,7 +141,8 @@ export const encryptMetadata = async (metadata: unknown, keyB64: string) => * * @returns The decrypted bytes. */ -export const decryptAssociatedData = ( encryptedData: Uint8Array, +export const decryptAssociatedData = ( + encryptedData: Uint8Array, headerB64: string, keyB64: string, ) => libsodium.decryptChaChaOneShot; diff --git a/web/packages/base/crypto/libsodium.ts b/web/packages/base/crypto/libsodium.ts index cf30256a86..f8be7c147c 100644 --- a/web/packages/base/crypto/libsodium.ts +++ b/web/packages/base/crypto/libsodium.ts @@ -11,6 +11,7 @@ import { mergeUint8Arrays } from "@/utils/array"; import { CustomError } from "@ente/shared/error"; import sodium, { type StateAddress } from "libsodium-wrappers"; +import type { DecryptBytes, EncryptBytes, EncryptedBytes } from "./types"; /** * Convert bytes ({@link Uint8Array}) to a base64 string. @@ -113,7 +114,7 @@ export async function fromHex(input: string) { } /** - * Encrypt the given {@link data} using the given (base64 encoded) key. + * Encrypt the given data using the given (base64 encoded) key. * * Use {@link decryptChaChaOneShot} to decrypt the result. * @@ -138,7 +139,7 @@ export async function fromHex(input: string) { * * See: https://doc.libsodium.org/secret-key_cryptography/secretbox * - * The difference here is that this function is meant to used for data + * The difference to those is that this function is meant to used for data * associated with a file (or some other Ente object, like a collection or an * entity). There is no technical reason to do it that way, just this way all * data associated with a file, including its actual contents, use the same @@ -155,10 +156,10 @@ export async function fromHex(input: string) { * encoded string). Both these values are needed to decrypt the data. The header * does not need to be secret. */ -export const encryptChaChaOneShot = async ( - data: Uint8Array, - keyB64: string, -) => { +export const encryptChaChaOneShot = async ({ + data, + keyB64, +}: EncryptBytes): Promise => { await sodium.ready; const uintkey: Uint8Array = await fromB64(keyB64); @@ -254,27 +255,15 @@ export async function encryptFileChunk( /** * Decrypt the result of {@link encryptChaChaOneShot}. - * - * @param encryptedData A {@link Uint8Array} containing the bytes to decrypt. - * - * @param header A base64 string containing the bytes of the decryption header - * that was produced during encryption. - * - * @param keyB64 The base64 string containing the key that was used to encrypt - * the data. - * - * @returns The decrypted bytes. - * - * @returns The decrypted metadata bytes. */ -export const decryptChaChaOneShot = async ( - encryptedData: Uint8Array, - headerB64: string, - keyB64: string, -) => { +export const decryptChaChaOneShot = async ({ + encryptedData, + decryptionHeaderB64, + keyB64, +}: DecryptBytes): Promise => { await sodium.ready; const pullState = sodium.crypto_secretstream_xchacha20poly1305_init_pull( - await fromB64(headerB64), + await fromB64(decryptionHeaderB64), await fromB64(keyB64), ); const pullResult = sodium.crypto_secretstream_xchacha20poly1305_pull( diff --git a/web/packages/base/crypto/types.ts b/web/packages/base/crypto/types.ts index 1411259c4d..a1719ff0c7 100644 --- a/web/packages/base/crypto/types.ts +++ b/web/packages/base/crypto/types.ts @@ -29,6 +29,45 @@ export interface EncryptJSON { keyB64: string; } +/** + * The result of encryption using the stream APIs used in one-shot mode. + * + * The encrypted data (bytes) and decryption header pair (base64 encoded + * string). Both these values are needed to decrypt the data. The header does + * not need to be secret. + */ +export interface EncryptedBytes { + /** + * A {@link Uint8Array} containing the encrypted data. + */ + encryptedData: Uint8Array; + /** + * A base64 string containing the decryption header. + * + * The header contains a random nonce and other libsodium specific metadata. + * It does not need to be secret, but it is required to decrypt the data. + */ + decryptionHeaderB64: string; +} + +/** + * The result of encryption using the stream APIs used in one-shot mode, with + * the encrypted data encoded as a base64 string. + */ +export interface EncryptedB64 { + /** + * A base64 string containing the encrypted data. + */ + encryptedDataB64: string; + /** + * A base64 string containing the decryption header. + * + * The header contains a random nonce and other libsodium specific metadata. + * It does not need to be secret, but it is required to decrypt the data. + */ + decryptionHeaderB64: string; +} + /** * A decryption request with the encrypted data as a base64 encoded string. */ From d6bf2437011322c8bfd3d6ac4bbe80fd8917c2cf Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 17:47:11 +0530 Subject: [PATCH 152/211] More --- web/packages/base/crypto/ente-impl.ts | 50 +++++------- web/packages/base/crypto/ente.ts | 105 ++++++-------------------- web/packages/base/crypto/types.ts | 16 ++-- 3 files changed, 51 insertions(+), 120 deletions(-) diff --git a/web/packages/base/crypto/ente-impl.ts b/web/packages/base/crypto/ente-impl.ts index 7d23f01f60..c1244a6853 100644 --- a/web/packages/base/crypto/ente-impl.ts +++ b/web/packages/base/crypto/ente-impl.ts @@ -1,6 +1,6 @@ /** Careful when adding add other imports! */ import * as libsodium from "./libsodium"; -import type { EncryptBytes, EncryptJSON } from "./types"; +import type { DecryptB64, EncryptBytes, EncryptJSON } from "./types"; export const _encryptAssociatedData = libsodium.encryptChaChaOneShot; @@ -29,41 +29,31 @@ export const _encryptMetadata = async ({ jsonValue, keyB64 }: EncryptJSON) => { export const _decryptAssociatedData = libsodium.decryptChaChaOneShot; -export const decryptThumbnailI = decryptAssociatedDataI; +export const _decryptThumbnail = _decryptAssociatedData; -export const decryptFileEmbeddingI = async ( - encryptedDataB64: string, - decryptionHeaderB64: string, - keyB64: string, -) => - decryptAssociatedDataI( - await libsodium.fromB64(encryptedDataB64), +export const _decryptFileEmbedding = async ({ + encryptedDataB64, + decryptionHeaderB64, + keyB64, +}: DecryptB64) => + _decryptAssociatedData({ + encryptedData: await libsodium.fromB64(encryptedDataB64), decryptionHeaderB64, keyB64, - ); + }); -export const decryptMetadataI = async ( - encryptedDataB64: string, - decryptionHeaderB64: string, - keyB64: string, -) => +export const _decryptMetadata = async (r: DecryptB64) => JSON.parse( - new TextDecoder().decode( - await decryptMetadataBytesI( - encryptedDataB64, - decryptionHeaderB64, - keyB64, - ), - ), + new TextDecoder().decode(await _decryptMetadataBytes(r)), ) as unknown; -export const decryptMetadataBytesI = async ( - encryptedDataB64: string, - decryptionHeaderB64: string, - keyB64: string, -) => - await decryptAssociatedDataI( - await libsodium.fromB64(encryptedDataB64), +export const _decryptMetadataBytes = async ({ + encryptedDataB64, + decryptionHeaderB64, + keyB64, +}: DecryptB64) => + await _decryptAssociatedData({ + encryptedData: await libsodium.fromB64(encryptedDataB64), decryptionHeaderB64, keyB64, - ); + }); diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index 7897288cbb..3fe281d776 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -50,8 +50,12 @@ import { assertionFailed } from "../assert"; import { inWorker } from "../env"; import * as ei from "./ente-impl"; -import * as libsodium from "./libsodium"; -import type { EncryptBytes } from "./types"; +import type { + DecryptB64, + DecryptBytes, + EncryptBytes, + EncryptJSON, +} from "./types"; import { sharedCryptoWorker } from "./worker"; /** @@ -96,8 +100,8 @@ export const encryptThumbnail = (r: EncryptBytes) => * * Use {@link decryptFileEmbedding} to decrypt the result. */ -export const encryptFileEmbedding = async (data: Uint8Array, keyB64: string) => - assertInWorker(ei.encryptFileEmbeddingI(data, keyB64)); +export const encryptFileEmbedding = async (r: EncryptBytes) => + assertInWorker(ei._encryptFileEmbedding(r)); /** * Encrypt the metadata associated with an Ente object (file, collection or @@ -113,15 +117,9 @@ export const encryptFileEmbedding = async (data: Uint8Array, keyB64: string) => * encrypted bytes, it returns their base64 string representation. * * Use {@link decryptMetadata} to decrypt the result. - * - * @param metadata The JSON value to encrypt. It can be an arbitrary JSON value, - * but since TypeScript currently doesn't have a native JSON type, it is typed - * as an unknown. - * - * @returns The encrypted data and decryption header, both as base64 strings. */ -export const encryptMetadata = async (metadata: unknown, keyB64: string) => - assertInWorker(ei.encryptMetadataI(metadata, keyB64)); +export const encryptMetadata = async (r: EncryptJSON) => + assertInWorker(ei._encryptMetadata(r)); /** * Decrypt arbitrary data associated with an Ente object (file, collection or @@ -130,53 +128,25 @@ export const encryptMetadata = async (metadata: unknown, keyB64: string) => * This is the sibling of {@link encryptAssociatedData}. * * See {@link decryptChaChaOneShot} for the implementation details. - * - * @param encryptedData A {@link Uint8Array} containing the bytes to decrypt. - * - * @param headerB64 A base64 string containing the decryption header that was - * produced during encryption. - * - * @param keyB64 A base64 string containing the encryption key. This is expected - * to be the key of the object to which {@link encryptedDataB64} is associated. - * - * @returns The decrypted bytes. */ -export const decryptAssociatedData = ( - encryptedData: Uint8Array, - headerB64: string, - keyB64: string, -) => libsodium.decryptChaChaOneShot; +export const decryptAssociatedData = (r: DecryptBytes) => + assertInWorker(ei._decryptAssociatedData(r)); /** * Decrypt the thumbnail for a file. * - * This is just an alias for {@link decryptAssociatedData}. + * This is the sibling of {@link encryptThumbnail}. */ -export const decryptThumbnail = decryptAssociatedData; +export const decryptThumbnail = (r: DecryptBytes) => + assertInWorker(ei._decryptThumbnail(r)); /** * Decrypt the embedding associated with a file using the file's key. * * This is the sibling of {@link encryptFileEmbedding}. - * - * @param encryptedDataB64 A base64 string containing the encrypted embedding. - * - * @param headerB64 A base64 string containing the decryption header produced - * during encryption. - * - * @param keyB64 A base64 string containing the encryption key. This is expected - * to be the key of the file with which {@link encryptedDataB64} is associated. - * - * @returns The decrypted metadata JSON object. */ -export const decryptFileEmbedding = async ( - encryptedDataB64: string, - decryptionHeaderB64: string, - keyB64: string, -) => - assertInWorker( - ei.decryptFileEmbeddingI(encryptedDataB64, decryptionHeaderB64, keyB64), - ); +export const decryptFileEmbedding = async (r: DecryptB64) => + assertInWorker(ei._decryptFileEmbedding(r)); /** * Decrypt the metadata associated with an Ente object (file, collection or @@ -184,49 +154,20 @@ export const decryptFileEmbedding = async ( * * This is the sibling of {@link encryptMetadata}. * - * @param encryptedDataB64 base64 encoded string containing the encrypted data. - * - * @param headerB64 base64 encoded string containing the decryption header - * produced during encryption. - * - * @param keyB64 base64 encoded string containing the encryption key. This is - * expected to be the key of the object with which {@link encryptedDataB64} is - * associated. - * * @returns The decrypted JSON value. Since TypeScript does not have a native * JSON type, we need to return it as an `unknown`. */ -export const decryptMetadata = async ( - encryptedDataB64: string, - decryptionHeaderB64: string, - keyB64: string, -) => +export const decryptMetadata = (r: DecryptB64) => inWorker() - ? ei.decryptMetadataI(encryptedDataB64, decryptionHeaderB64, keyB64) - : sharedCryptoWorker().then((w) => - w.decryptMetadata(encryptedDataB64, decryptionHeaderB64, keyB64), - ); + ? ei._decryptMetadata(r) + : sharedCryptoWorker().then((w) => w.decryptMetadata(r)); /** * A variant of {@link decryptMetadata} that does not attempt to parse the * decrypted data as a JSON string and instead just returns the raw decrypted * bytes that we got. */ -export const decryptMetadataBytes = ( - encryptedDataB64: string, - decryptionHeaderB64: string, - keyB64: string, -) => +export const decryptMetadataBytes = (r: DecryptB64) => inWorker() - ? ei.decryptMetadataBytesI( - encryptedDataB64, - decryptionHeaderB64, - keyB64, - ) - : sharedCryptoWorker().then((w) => - w.decryptMetadataBytes( - encryptedDataB64, - decryptionHeaderB64, - keyB64, - ), - ); + ? ei._decryptMetadataBytes(r) + : sharedCryptoWorker().then((w) => w.decryptMetadataBytes(r)); diff --git a/web/packages/base/crypto/types.ts b/web/packages/base/crypto/types.ts index a1719ff0c7..edd2db3241 100644 --- a/web/packages/base/crypto/types.ts +++ b/web/packages/base/crypto/types.ts @@ -69,13 +69,13 @@ export interface EncryptedB64 { } /** - * A decryption request with the encrypted data as a base64 encoded string. + * A decryption request with the encrypted data as bytes. */ -export interface DecryptB64 { +export interface DecryptBytes { /** - * A base64 string containing the data to decrypt. + * A {@link Uint8Array} containing the bytes to decrypt. */ - encryptedDataB64: string; + encryptedData: Uint8Array; /** * A base64 string containing the decryption header that was produced during * encryption. @@ -91,13 +91,13 @@ export interface DecryptB64 { } /** - * A decryption request with the encrypted data as bytes. + * A decryption request with the encrypted data as a base64 encoded string. */ -export interface DecryptBytes { +export interface DecryptB64 { /** - * A {@link Uint8Array} containing the bytes to decrypt. + * A base64 string containing the data to decrypt. */ - encryptedData: Uint8Array; + encryptedDataB64: string; /** * A base64 string containing the decryption header that was produced during * encryption. From 50d7d7e9a19aeec6d22d4df26fbd0bac09f4cee3 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 17:55:39 +0530 Subject: [PATCH 153/211] Proxy --- web/packages/base/crypto/worker/worker.ts | 24 +++++------------------ 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/web/packages/base/crypto/worker/worker.ts b/web/packages/base/crypto/worker/worker.ts index ab775b6a61..eae03d6258 100644 --- a/web/packages/base/crypto/worker/worker.ts +++ b/web/packages/base/crypto/worker/worker.ts @@ -12,25 +12,11 @@ import * as libsodium from "../libsodium"; * Note: Keep these methods logic free. They are meant to be trivial proxies. */ export class CryptoWorker { - async encryptThumbnail(a: Uint8Array, b: string) { - return ei.encryptThumbnailI(a, b); - } - - async encryptMetadata(a: unknown, b: string) { - return ei.encryptMetadataI(a, b); - } - - async decryptThumbnail(a: Uint8Array, b: string, c: string) { - return ei.decryptThumbnailI(a, b, c); - } - - async decryptMetadata(a: string, b: string, c: string) { - return ei.decryptMetadataI(a, b, c); - } - - async decryptMetadataBytes(a: string, b: string, c: string) { - return ei.decryptMetadataBytesI(a, b, c); - } + encryptThumbnail = ei._encryptThumbnail; + encryptMetadata = ei._encryptMetadata; + decryptThumbnail = ei._decryptThumbnail; + decryptMetadata = ei._decryptMetadata; + decryptMetadataBytes = ei._decryptMetadataBytes; // TODO: -- AUDIT BELOW -- From e29c9288c08fb6f3e625aeca02553ca0abfa0df6 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 18:02:44 +0530 Subject: [PATCH 154/211] Abstract --- web/packages/base/crypto/ente-impl.ts | 50 +++++++++++---------------- 1 file changed, 21 insertions(+), 29 deletions(-) diff --git a/web/packages/base/crypto/ente-impl.ts b/web/packages/base/crypto/ente-impl.ts index c1244a6853..704e41d7f8 100644 --- a/web/packages/base/crypto/ente-impl.ts +++ b/web/packages/base/crypto/ente-impl.ts @@ -1,42 +1,43 @@ /** Careful when adding add other imports! */ import * as libsodium from "./libsodium"; -import type { DecryptB64, EncryptBytes, EncryptJSON } from "./types"; +import type { + DecryptB64, + EncryptBytes, + EncryptedB64, + EncryptedBytes, + EncryptJSON, +} from "./types"; + +const EncryptedBytesToB64 = async ({ + encryptedData, + decryptionHeaderB64, +}: EncryptedBytes): Promise => ({ + encryptedDataB64: await libsodium.toB64(encryptedData), + decryptionHeaderB64, +}); export const _encryptAssociatedData = libsodium.encryptChaChaOneShot; export const _encryptThumbnail = _encryptAssociatedData; -export const _encryptFileEmbedding = async (r: EncryptBytes) => { - const { encryptedData, decryptionHeaderB64 } = - await _encryptAssociatedData(r); - return { - encryptedDataB64: await libsodium.toB64(encryptedData), - decryptionHeaderB64, - }; -}; +export const _encryptFileEmbedding = (r: EncryptBytes) => + _encryptAssociatedData(r).then(EncryptedBytesToB64); export const _encryptMetadata = async ({ jsonValue, keyB64 }: EncryptJSON) => { const data = new TextEncoder().encode(JSON.stringify(jsonValue)); - - const { encryptedData, decryptionHeaderB64 } = await _encryptAssociatedData( - { data, keyB64 }, - ); - return { - encryptedDataB64: await libsodium.toB64(encryptedData), - decryptionHeaderB64, - }; + return EncryptedBytesToB64(await _encryptAssociatedData({ data, keyB64 })); }; export const _decryptAssociatedData = libsodium.decryptChaChaOneShot; export const _decryptThumbnail = _decryptAssociatedData; -export const _decryptFileEmbedding = async ({ +export const _decryptMetadataBytes = async ({ encryptedDataB64, decryptionHeaderB64, keyB64, }: DecryptB64) => - _decryptAssociatedData({ + await _decryptAssociatedData({ encryptedData: await libsodium.fromB64(encryptedDataB64), decryptionHeaderB64, keyB64, @@ -47,13 +48,4 @@ export const _decryptMetadata = async (r: DecryptB64) => new TextDecoder().decode(await _decryptMetadataBytes(r)), ) as unknown; -export const _decryptMetadataBytes = async ({ - encryptedDataB64, - decryptionHeaderB64, - keyB64, -}: DecryptB64) => - await _decryptAssociatedData({ - encryptedData: await libsodium.fromB64(encryptedDataB64), - decryptionHeaderB64, - keyB64, - }); +export const _decryptFileEmbedding = _decryptMetadataBytes; From 95cfdc4eaacc055ae1f3d84bcdd7676682a3df2a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 19:11:24 +0530 Subject: [PATCH 155/211] Rearrange --- web/packages/base/crypto/ente-impl.ts | 13 ++++++++----- web/packages/base/crypto/ente.ts | 22 +++++++++++++++------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/web/packages/base/crypto/ente-impl.ts b/web/packages/base/crypto/ente-impl.ts index 704e41d7f8..40521055a1 100644 --- a/web/packages/base/crypto/ente-impl.ts +++ b/web/packages/base/crypto/ente-impl.ts @@ -20,13 +20,16 @@ export const _encryptAssociatedData = libsodium.encryptChaChaOneShot; export const _encryptThumbnail = _encryptAssociatedData; -export const _encryptFileEmbedding = (r: EncryptBytes) => +export const _encryptMetadataBytes = (r: EncryptBytes) => _encryptAssociatedData(r).then(EncryptedBytesToB64); -export const _encryptMetadata = async ({ jsonValue, keyB64 }: EncryptJSON) => { - const data = new TextEncoder().encode(JSON.stringify(jsonValue)); - return EncryptedBytesToB64(await _encryptAssociatedData({ data, keyB64 })); -}; +export const _encryptFileEmbedding = _encryptMetadataBytes; + +export const _encryptMetadataJSON = ({ jsonValue, keyB64 }: EncryptJSON) => + _encryptMetadataBytes({ + data: new TextEncoder().encode(JSON.stringify(jsonValue)), + keyB64, + }); export const _decryptAssociatedData = libsodium.decryptChaChaOneShot; diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index 3fe281d776..e3f99dcfa9 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -86,17 +86,25 @@ export const encryptAssociatedData = (r: EncryptBytes) => * Encrypt the thumbnail for a file. * * This is just an alias for {@link encryptAssociatedData}. + * + * Use {@link decryptFileEmbedding} to decrypt the result. */ export const encryptThumbnail = (r: EncryptBytes) => assertInWorker(ei._encryptThumbnail(r)); +/** + * A variant of {@link encryptAssociatedData} that returns the encrypted data in + * the result as a base64 string instead of its bytes. + * + * Use {@link decryptMetadata} to decrypt the result. + */ +export const encryptMetadataBytes = (r: EncryptBytes) => + assertInWorker(ei._encryptMetadataBytes(r)); + /** * Encrypted the embedding associated with a file using the file's key. * - * This as a variant of {@link encryptAssociatedData} tailored for - * encrypting the embeddings (a.k.a. derived data) associated with a file. In - * particular, it returns the encrypted data in the result as a base64 string - * instead of its bytes. + * This is just an alias for {@link encryptMetadataBytes}. * * Use {@link decryptFileEmbedding} to decrypt the result. */ @@ -104,7 +112,7 @@ export const encryptFileEmbedding = async (r: EncryptBytes) => assertInWorker(ei._encryptFileEmbedding(r)); /** - * Encrypt the metadata associated with an Ente object (file, collection or + * Encrypt the JSON metadata associated with an Ente object (file, collection or * entity) using the object's key. * * This is a variant of {@link encryptAssociatedData} tailored for encrypting @@ -118,8 +126,8 @@ export const encryptFileEmbedding = async (r: EncryptBytes) => * * Use {@link decryptMetadata} to decrypt the result. */ -export const encryptMetadata = async (r: EncryptJSON) => - assertInWorker(ei._encryptMetadata(r)); +export const encryptMetadataJSON = async (r: EncryptJSON) => + assertInWorker(ei._encryptMetadataJSON(r)); /** * Decrypt arbitrary data associated with an Ente object (file, collection or From ff7718a878989216834a91e8341493ba39b8b807 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 19:17:23 +0530 Subject: [PATCH 156/211] Fin --- web/packages/base/crypto/ente-impl.ts | 6 ++-- web/packages/base/crypto/ente.ts | 35 +++++++++++------------ web/packages/base/crypto/worker/worker.ts | 4 +-- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/web/packages/base/crypto/ente-impl.ts b/web/packages/base/crypto/ente-impl.ts index 40521055a1..831dab029c 100644 --- a/web/packages/base/crypto/ente-impl.ts +++ b/web/packages/base/crypto/ente-impl.ts @@ -46,9 +46,9 @@ export const _decryptMetadataBytes = async ({ keyB64, }); -export const _decryptMetadata = async (r: DecryptB64) => +export const _decryptFileEmbedding = _decryptMetadataBytes; + +export const _decryptMetadataJSON = async (r: DecryptB64) => JSON.parse( new TextDecoder().decode(await _decryptMetadataBytes(r)), ) as unknown; - -export const _decryptFileEmbedding = _decryptMetadataBytes; diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index e3f99dcfa9..e4f1d64298 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -96,7 +96,7 @@ export const encryptThumbnail = (r: EncryptBytes) => * A variant of {@link encryptAssociatedData} that returns the encrypted data in * the result as a base64 string instead of its bytes. * - * Use {@link decryptMetadata} to decrypt the result. + * Use {@link decryptMetadataBytes} to decrypt the result. */ export const encryptMetadataBytes = (r: EncryptBytes) => assertInWorker(ei._encryptMetadataBytes(r)); @@ -124,7 +124,7 @@ export const encryptFileEmbedding = async (r: EncryptBytes) => * encodes into a string, and encrypts that. And instead of returning the raw * encrypted bytes, it returns their base64 string representation. * - * Use {@link decryptMetadata} to decrypt the result. + * Use {@link decryptMetadataJSON} to decrypt the result. */ export const encryptMetadataJSON = async (r: EncryptJSON) => assertInWorker(ei._encryptMetadataJSON(r)); @@ -148,6 +148,16 @@ export const decryptAssociatedData = (r: DecryptBytes) => export const decryptThumbnail = (r: DecryptBytes) => assertInWorker(ei._decryptThumbnail(r)); +/** + * Decrypt metadata associated with an Ente object. + * + * This is the sibling of {@link decryptMetadataBytes}. + */ +export const decryptMetadataBytes = (r: DecryptB64) => + inWorker() + ? ei._decryptMetadataBytes(r) + : sharedCryptoWorker().then((w) => w.decryptMetadataBytes(r)); + /** * Decrypt the embedding associated with a file using the file's key. * @@ -157,25 +167,14 @@ export const decryptFileEmbedding = async (r: DecryptB64) => assertInWorker(ei._decryptFileEmbedding(r)); /** - * Decrypt the metadata associated with an Ente object (file, collection or - * entity) using the object's key. + * Decrypt the metadata JSON associated with an Ente object. * - * This is the sibling of {@link encryptMetadata}. + * This is the sibling of {@link encryptMetadataJSON}. * * @returns The decrypted JSON value. Since TypeScript does not have a native * JSON type, we need to return it as an `unknown`. */ -export const decryptMetadata = (r: DecryptB64) => - inWorker() - ? ei._decryptMetadata(r) - : sharedCryptoWorker().then((w) => w.decryptMetadata(r)); - -/** - * A variant of {@link decryptMetadata} that does not attempt to parse the - * decrypted data as a JSON string and instead just returns the raw decrypted - * bytes that we got. - */ -export const decryptMetadataBytes = (r: DecryptB64) => +export const decryptMetadataJSON = (r: DecryptB64) => inWorker() - ? ei._decryptMetadataBytes(r) - : sharedCryptoWorker().then((w) => w.decryptMetadataBytes(r)); + ? ei._decryptMetadataJSON(r) + : sharedCryptoWorker().then((w) => w.decryptMetadataJSON(r)); diff --git a/web/packages/base/crypto/worker/worker.ts b/web/packages/base/crypto/worker/worker.ts index eae03d6258..ad86690d02 100644 --- a/web/packages/base/crypto/worker/worker.ts +++ b/web/packages/base/crypto/worker/worker.ts @@ -13,9 +13,9 @@ import * as libsodium from "../libsodium"; */ export class CryptoWorker { encryptThumbnail = ei._encryptThumbnail; - encryptMetadata = ei._encryptMetadata; + encryptMetadataJSON = ei._encryptMetadataJSON; decryptThumbnail = ei._decryptThumbnail; - decryptMetadata = ei._decryptMetadata; + decryptMetadataJSON = ei._decryptMetadataJSON; decryptMetadataBytes = ei._decryptMetadataBytes; // TODO: -- AUDIT BELOW -- From 041ad135c966c6d2b0823384d2940e4297c090e0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 19:25:49 +0530 Subject: [PATCH 157/211] Use during upload --- .../src/services/upload/uploadManager.ts | 19 +++---- .../src/services/upload/uploadService.ts | 49 ++++++++++--------- web/packages/base/crypto/worker/index.ts | 17 +++++-- 3 files changed, 49 insertions(+), 36 deletions(-) diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 571ce34b9a..0ceff98c06 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -1,3 +1,7 @@ +import { + createComlinkCryptoWorker, + type CryptoWorker, +} from "@/base/crypto/worker"; import { ensureElectron } from "@/base/electron"; import { lowercaseExtension, nameAndExtension } from "@/base/file"; import log from "@/base/log"; @@ -16,11 +20,8 @@ import { import { EncryptedEnteFile, EnteFile } from "@/new/photos/types/file"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; -import { getDedicatedCryptoWorker } from "@ente/shared/crypto"; -import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { CustomError } from "@ente/shared/error"; import { Canceler } from "axios"; -import type { Remote } from "comlink"; import isElectron from "is-electron"; import { getLocalPublicFiles, @@ -316,8 +317,8 @@ const groupByResult = (finishedUploads: FinishedUploads) => { }; class UploadManager { - private cryptoWorkers = new Array< - ComlinkWorker + private comlinkCryptoWorkers = new Array< + ComlinkWorker >(maxConcurrentUploads); private parsedMetadataJSONMap: Map; private itemsToBeUploaded: ClusteredUploadItem[]; @@ -448,7 +449,7 @@ class UploadManager { this.uiService.setUploadStage(UPLOAD_STAGES.FINISH); void globalThis.electron?.clearPendingUploads(); for (let i = 0; i < maxConcurrentUploads; i++) { - this.cryptoWorkers[i]?.terminate(); + this.comlinkCryptoWorkers[i]?.terminate(); } this.uploadInProgress = false; clearInterval(logInterval); @@ -510,14 +511,14 @@ class UploadManager { i < maxConcurrentUploads && this.itemsToBeUploaded.length > 0; i++ ) { - this.cryptoWorkers[i] = getDedicatedCryptoWorker(); - const worker = await this.cryptoWorkers[i].remote; + this.comlinkCryptoWorkers[i] = createComlinkCryptoWorker(); + const worker = await this.comlinkCryptoWorkers[i].remote; uploadProcesses.push(this.uploadNextItemInQueue(worker)); } await Promise.all(uploadProcesses); } - private async uploadNextItemInQueue(worker: Remote) { + private async uploadNextItemInQueue(worker: CryptoWorker) { const uiService = this.uiService; while (this.itemsToBeUploaded.length > 0) { diff --git a/web/apps/photos/src/services/upload/uploadService.ts b/web/apps/photos/src/services/upload/uploadService.ts index a7b0f5c477..42c2706105 100644 --- a/web/apps/photos/src/services/upload/uploadService.ts +++ b/web/apps/photos/src/services/upload/uploadService.ts @@ -1,3 +1,8 @@ +import { + ENCRYPTION_CHUNK_SIZE, + type B64EncryptionResult, +} from "@/base/crypto/libsodium"; +import { type CryptoWorker } from "@/base/crypto/worker"; import { ensureElectron } from "@/base/electron"; import { basename } from "@/base/file"; import log from "@/base/log"; @@ -29,11 +34,7 @@ import { EncryptedMagicMetadata } from "@/new/photos/types/magicMetadata"; import { detectFileTypeInfoFromChunk } from "@/new/photos/utils/detect-type"; import { readStream } from "@/new/photos/utils/native-stream"; import { ensure } from "@/utils/ensure"; -import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; -import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; -import { ENCRYPTION_CHUNK_SIZE } from "@ente/shared/crypto/internal/libsodium"; import { CustomError, handleUploadError } from "@ente/shared/error"; -import type { Remote } from "comlink"; import { addToCollection } from "services/collectionService"; import { PublicUploadProps, @@ -320,7 +321,7 @@ export const uploader = async ( uploaderName: string, existingFiles: EnteFile[], parsedMetadataJSONMap: Map, - worker: Remote, + worker: CryptoWorker, isCFUploadProxyDisabled: boolean, abortIfCancelled: () => void, makeProgessTracker: MakeProgressTracker, @@ -681,7 +682,7 @@ const extractAssetMetadata = async ( lastModifiedMs: number, collectionID: number, parsedMetadataJSONMap: Map, - worker: Remote, + worker: CryptoWorker, ): Promise => isLivePhoto ? await extractLivePhotoMetadata( @@ -707,7 +708,7 @@ const extractLivePhotoMetadata = async ( lastModifiedMs: number, collectionID: number, parsedMetadataJSONMap: Map, - worker: Remote, + worker: CryptoWorker, ) => { const imageFileTypeInfo: FileTypeInfo = { fileType: FileType.image, @@ -744,7 +745,7 @@ const extractImageOrVideoMetadata = async ( lastModifiedMs: number, collectionID: number, parsedMetadataJSONMap: Map, - worker: Remote, + worker: CryptoWorker, ) => { const fileName = uploadItemFileName(uploadItem); const { fileType } = fileTypeInfo; @@ -845,10 +846,7 @@ const tryExtractVideoMetadata = async (uploadItem: UploadItem) => { } }; -const computeHash = async ( - uploadItem: UploadItem, - worker: Remote, -) => { +const computeHash = async (uploadItem: UploadItem, worker: CryptoWorker) => { const { stream, chunkCount } = await readUploadItem(uploadItem); const hashState = await worker.initChunkHashing(); @@ -1112,7 +1110,7 @@ const constructPublicMagicMetadata = async ( const encryptFile = async ( file: FileWithMetadata, encryptionKey: string, - worker: Remote, + worker: CryptoWorker, ): Promise => { const { key: fileKey, file: encryptedFiledata } = await encryptFiledata( file.fileStreamOrData, @@ -1122,23 +1120,26 @@ const encryptFile = async ( const { encryptedData: thumbEncryptedData, decryptionHeaderB64: thumbDecryptionHeader, - } = await worker.encryptThumbnail(file.thumbnail, fileKey); + } = await worker.encryptThumbnail({ + data: file.thumbnail, + keyB64: fileKey, + }); const encryptedThumbnail = { encryptedData: thumbEncryptedData, decryptionHeader: thumbDecryptionHeader, }; - const encryptedMetadata = await worker.encryptMetadata( - file.metadata, - fileKey, - ); + const encryptedMetadata = await worker.encryptMetadataJSON({ + jsonValue: file.metadata, + keyB64: fileKey, + }); let encryptedPubMagicMetadata: EncryptedMagicMetadata; if (file.pubMagicMetadata) { - const encryptedPubMagicMetadataData = await worker.encryptMetadata( - file.pubMagicMetadata.data, - fileKey, - ); + const encryptedPubMagicMetadataData = await worker.encryptMetadataJSON({ + jsonValue: file.pubMagicMetadata.data, + keyB64: fileKey, + }); encryptedPubMagicMetadata = { version: file.pubMagicMetadata.version, count: file.pubMagicMetadata.count, @@ -1164,7 +1165,7 @@ const encryptFile = async ( const encryptFiledata = async ( fileStreamOrData: FileStream | Uint8Array, - worker: Remote, + worker: CryptoWorker, ): Promise> => fileStreamOrData instanceof Uint8Array ? await worker.encryptFile(fileStreamOrData) @@ -1172,7 +1173,7 @@ const encryptFiledata = async ( const encryptFileStream = async ( fileData: FileStream, - worker: Remote, + worker: CryptoWorker, ) => { const { stream, chunkCount } = fileData; const fileStreamReader = stream.getReader(); diff --git a/web/packages/base/crypto/worker/index.ts b/web/packages/base/crypto/worker/index.ts index 040250c286..2443080aaf 100644 --- a/web/packages/base/crypto/worker/index.ts +++ b/web/packages/base/crypto/worker/index.ts @@ -1,16 +1,27 @@ import { ComlinkWorker } from "@/base/worker/comlink-worker"; import type { CryptoWorker } from "./worker"; -/** Cached instance of the {@link ComlinkWorker} that wraps our web worker. */ +/** + * Reexport the type for easier import in call sites. + */ +export { CryptoWorker } from "./worker"; + +/** + * Cached instance of the {@link ComlinkWorker} that wraps our web worker. + */ let _comlinkWorker: ComlinkWorker | undefined; /** * Lazily created, cached, instance of a CryptoWorker web worker. */ export const sharedCryptoWorker = async () => - (_comlinkWorker ??= createComlinkWorker()).remote; + (_comlinkWorker ??= createComlinkCryptoWorker()).remote; -const createComlinkWorker = () => +/** + * Create a new instance of a comlink worker that wraps a {@link CryptoWorker} + * web worker. + */ +export const createComlinkCryptoWorker = () => new ComlinkWorker( "Crypto", new Worker(new URL("worker.ts", import.meta.url)), From 8acc5ac62dd54273a8da6013b05f147f08c32391 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 19:50:36 +0530 Subject: [PATCH 158/211] Propagate --- web/apps/auth/src/services/remote.ts | 14 ++-- web/apps/cast/src/services/render.ts | 16 ++-- .../manage/linkPassword/setPassword.tsx | 4 +- .../photos/src/components/FixCreationTime.tsx | 4 +- .../components/PhotoViewer/FileInfo/index.tsx | 4 +- .../photos/src/pages/shared-albums/index.tsx | 6 +- .../photos/src/services/collectionService.ts | 80 +++++++++---------- web/apps/photos/src/services/entityService.ts | 16 ++-- web/apps/photos/src/services/fileService.ts | 22 ++--- .../src/services/publicCollectionService.ts | 15 ++-- web/apps/photos/src/utils/crypto/index.ts | 4 +- web/apps/photos/src/utils/file/index.ts | 34 ++++---- .../photos/src/utils/magicMetadata/index.ts | 4 +- .../accounts/pages/change-password.tsx | 4 +- web/packages/accounts/pages/credentials.tsx | 8 +- web/packages/accounts/pages/recover.tsx | 4 +- .../accounts/pages/two-factor/recover.tsx | 6 +- web/packages/accounts/services/passkey.ts | 9 +-- web/packages/accounts/services/srp.ts | 4 +- web/packages/accounts/utils/srp.ts | 4 +- web/packages/new/photos/services/download.ts | 18 ++--- .../components/VerifyMasterPasswordForm.tsx | 4 +- web/packages/shared/crypto/helpers.ts | 16 ++-- web/packages/shared/file-metadata.ts | 4 +- web/packages/shared/user/index.ts | 6 +- 25 files changed, 153 insertions(+), 157 deletions(-) diff --git a/web/apps/auth/src/services/remote.ts b/web/apps/auth/src/services/remote.ts index 758bd31b66..e30b8d8f90 100644 --- a/web/apps/auth/src/services/remote.ts +++ b/web/apps/auth/src/services/remote.ts @@ -1,7 +1,7 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import { ensureString } from "@/utils/ensure"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { ApiError, CustomError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; @@ -13,7 +13,7 @@ export const getAuthCodes = async (): Promise => { const masterKey = await getActualKey(); try { const authKeyData = await getAuthKey(); - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const authenticatorKey = await cryptoWorker.decryptB64( authKeyData.encryptedKey, authKeyData.header, @@ -30,11 +30,11 @@ export const getAuthCodes = async (): Promise => { if (!entity.header) return undefined; try { const decryptedCode = - await cryptoWorker.decryptMetadata( - entity.encryptedData, - entity.header, - authenticatorKey, - ); + await cryptoWorker.decryptMetadataJSON({ + encryptedDataB64: entity.encryptedData, + decryptionHeaderB64: entity.header, + keyB64: authenticatorKey, + }); return codeFromURIString( entity.id, ensureString(decryptedCode), diff --git a/web/apps/cast/src/services/render.ts b/web/apps/cast/src/services/render.ts index d512b039ef..1ba9aa502b 100644 --- a/web/apps/cast/src/services/render.ts +++ b/web/apps/cast/src/services/render.ts @@ -5,6 +5,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { nameAndExtension } from "@/base/file"; import log from "@/base/log"; import { apiURL, customAPIOrigin } from "@/base/origins"; @@ -21,7 +22,6 @@ import type { import { shuffled } from "@/utils/array"; import { ensure } from "@/utils/ensure"; import { wait } from "@/utils/promise"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { ApiError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import type { AxiosResponse } from "axios"; @@ -188,7 +188,7 @@ const decryptEnteFile = async ( encryptedFile: EncryptedEnteFile, collectionKey: string, ): Promise => { - const worker = await ComlinkCryptoWorker.getInstance(); + const worker = await sharedCryptoWorker(); const { encryptedKey, keyDecryptionNonce, @@ -202,11 +202,11 @@ const decryptEnteFile = async ( keyDecryptionNonce, collectionKey, ); - const fileMetadata = await worker.decryptMetadata( - metadata.encryptedData, - metadata.decryptionHeader, - fileKey, - ); + const fileMetadata = await worker.decryptMetadataJSON({ + encryptedDataB64: metadata.encryptedData, + decryptionHeaderB64: metadata.decryptionHeader, + keyB64: fileKey, + }); let fileMagicMetadata: FileMagicMetadata | undefined; let filePubMagicMetadata: FilePublicMagicMetadata | undefined; if (magicMetadata?.data) { @@ -351,7 +351,7 @@ const downloadFile = async ( `Failed to fetch file with ID ${file.id}: HTTP ${res.status}`, ); - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const decrypted = await cryptoWorker.decryptFile( new Uint8Array(await res.arrayBuffer()), await cryptoWorker.fromB64( diff --git a/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/setPassword.tsx b/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/setPassword.tsx index 2b8e1fbb96..ed33f8565a 100644 --- a/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/setPassword.tsx +++ b/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/setPassword.tsx @@ -1,7 +1,7 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import SingleInputForm, { type SingleInputFormProps, } from "@ente/shared/components/SingleInputForm"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { Dialog, Stack, Typography } from "@mui/material"; import { t } from "i18next"; @@ -27,7 +27,7 @@ export function PublicLinkSetPassword({ }; const enablePublicUrlPassword = async (password: string) => { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const kekSalt = await cryptoWorker.generateSaltToDeriveKey(); const kek = await cryptoWorker.deriveInteractiveKey(password, kekSalt); diff --git a/web/apps/photos/src/components/FixCreationTime.tsx b/web/apps/photos/src/components/FixCreationTime.tsx index 9018a48022..b5e3100ccc 100644 --- a/web/apps/photos/src/components/FixCreationTime.tsx +++ b/web/apps/photos/src/components/FixCreationTime.tsx @@ -1,3 +1,4 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { decryptPublicMagicMetadata, @@ -13,7 +14,6 @@ import { EnteFile } from "@/new/photos/types/file"; import { fileLogID } from "@/new/photos/utils/file"; import { ensure } from "@/utils/ensure"; import DialogBox from "@ente/shared/components/DialogBox/"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { Button, FormControl, @@ -353,7 +353,7 @@ const updateEnteFileDate = async ( if (!newDate) return; - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const existingUIDate = getUICreationDate( enteFile, diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index d724274640..896a82facf 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -1,5 +1,6 @@ import { EnteDrawer } from "@/base/components/EnteDrawer"; import { Titlebar } from "@/base/components/Titlebar"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { nameAndExtension } from "@/base/file"; import log from "@/base/log"; import type { ParsedMetadata } from "@/media/file-metadata"; @@ -19,7 +20,6 @@ import { formattedByteSize } from "@/new/photos/utils/units"; import CopyButton from "@ente/shared/components/CodeBlock/CopyButton"; import { FlexWrapper } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { getPublicMagicMetadataMTSync } from "@ente/shared/file-metadata"; import { formatDate, formatTime } from "@ente/shared/time/format"; import BackupOutlined from "@mui/icons-material/BackupOutlined"; @@ -399,7 +399,7 @@ export const CreationTime: React.FC = ({ return; } - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); await updateRemotePublicMagicMetadata( enteFile, { dateTime, editedTime: timestamp }, diff --git a/web/apps/photos/src/pages/shared-albums/index.tsx b/web/apps/photos/src/pages/shared-albums/index.tsx index c1715fa878..64cc325660 100644 --- a/web/apps/photos/src/pages/shared-albums/index.tsx +++ b/web/apps/photos/src/pages/shared-albums/index.tsx @@ -1,3 +1,4 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import downloadManager from "@/new/photos/services/download"; import { EnteFile } from "@/new/photos/types/file"; @@ -16,7 +17,6 @@ import SingleInputForm, { type SingleInputFormProps, } from "@ente/shared/components/SingleInputForm"; import { PHOTOS_PAGES as PAGES } from "@ente/shared/constants/pages"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { CustomError, parseSharingErrorCodes } from "@ente/shared/error"; import { useFileInput } from "@ente/shared/hooks/useFileInput"; import AddPhotoAlternateOutlined from "@mui/icons-material/AddPhotoAlternateOutlined"; @@ -204,7 +204,7 @@ export default function PublicCollectionGallery() { const main = async () => { let redirectingToWebsite = false; try { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); await downloadManager.init(); url.current = window.location.href; @@ -421,7 +421,7 @@ export default function PublicCollectionGallery() { setFieldError, ) => { try { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); let hashedPassword: string = null; try { const publicUrl = publicCollection.publicURLs[0]; diff --git a/web/apps/photos/src/services/collectionService.ts b/web/apps/photos/src/services/collectionService.ts index e98ee93bde..9a25b8a9cb 100644 --- a/web/apps/photos/src/services/collectionService.ts +++ b/web/apps/photos/src/services/collectionService.ts @@ -1,3 +1,4 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import { ItemVisibility } from "@/media/file-metadata"; @@ -9,7 +10,6 @@ import { UpdateMagicMetadataRequest, } from "@/new/photos/types/magicMetadata"; import { batch } from "@/utils/array"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { CustomError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; @@ -99,7 +99,7 @@ const getCollectionWithSecrets = async ( collection: EncryptedCollection, masterKey: string, ): Promise => { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const userID = getData(LS_KEYS.USER).id; let collectionKey: string; if (collection.owner.id === userID) { @@ -133,22 +133,22 @@ const getCollectionWithSecrets = async ( if (collection.magicMetadata?.data) { collectionMagicMetadata = { ...collection.magicMetadata, - data: await cryptoWorker.decryptMetadata( - collection.magicMetadata.data, - collection.magicMetadata.header, - collectionKey, - ), + data: await cryptoWorker.decryptMetadataJSON({ + encryptedDataB64: collection.magicMetadata.data, + decryptionHeaderB64: collection.magicMetadata.header, + keyB64: collectionKey, + }), }; } let collectionPublicMagicMetadata: CollectionPublicMagicMetadata; if (collection.pubMagicMetadata?.data) { collectionPublicMagicMetadata = { ...collection.pubMagicMetadata, - data: await cryptoWorker.decryptMetadata( - collection.pubMagicMetadata.data, - collection.pubMagicMetadata.header, - collectionKey, - ), + data: await cryptoWorker.decryptMetadataJSON({ + encryptedDataB64: collection.pubMagicMetadata.data, + decryptionHeaderB64: collection.pubMagicMetadata.header, + keyB64: collectionKey, + }), }; } @@ -156,11 +156,11 @@ const getCollectionWithSecrets = async ( if (collection.sharedMagicMetadata?.data) { collectionShareeMagicMetadata = { ...collection.sharedMagicMetadata, - data: await cryptoWorker.decryptMetadata( - collection.sharedMagicMetadata.data, - collection.sharedMagicMetadata.header, - collectionKey, - ), + data: await cryptoWorker.decryptMetadataJSON({ + encryptedDataB64: collection.sharedMagicMetadata.data, + decryptionHeaderB64: collection.sharedMagicMetadata.header, + keyB64: collectionKey, + }), }; } @@ -415,7 +415,7 @@ const createCollection = async ( magicMetadataProps?: CollectionMagicMetadataProps, ): Promise => { try { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const encryptionKey = await getActualKey(); const token = getToken(); const collectionKey = await cryptoWorker.generateEncryptionKey(); @@ -427,10 +427,10 @@ const createCollection = async ( if (magicMetadataProps) { const magicMetadata = await updateMagicMetadata(magicMetadataProps); const encryptedMagicMetadataProps = - await cryptoWorker.encryptMetadata( - magicMetadataProps, - collectionKey, - ); + await cryptoWorker.encryptMetadataJSON({ + jsonValue: magicMetadataProps, + keyB64: collectionKey, + }); encryptedMagicMetadata = { ...magicMetadata, @@ -607,7 +607,7 @@ const encryptWithNewCollectionKey = async ( files: EnteFile[], ): Promise => { const fileKeysEncryptedWithNewCollection: EncryptedFileKey[] = []; - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); for (const file of files) { const newEncryptedKey = await cryptoWorker.encryptToB64( file.key, @@ -797,13 +797,13 @@ export const updateCollectionMagicMetadata = async ( return; } - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadata( - updatedMagicMetadata.data, - collection.key, - ); + await cryptoWorker.encryptMetadataJSON({ + jsonValue: updatedMagicMetadata.data, + keyB64: collection.key, + }); const reqBody: UpdateMagicMetadataRequest = { id: collection.id, @@ -842,13 +842,13 @@ export const updateSharedCollectionMagicMetadata = async ( return; } - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadata( - updatedMagicMetadata.data, - collection.key, - ); + await cryptoWorker.encryptMetadataJSON({ + jsonValue: updatedMagicMetadata.data, + keyB64: collection.key, + }); const reqBody: UpdateMagicMetadataRequest = { id: collection.id, @@ -887,13 +887,13 @@ export const updatePublicCollectionMagicMetadata = async ( return; } - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadata( - updatedPublicMagicMetadata.data, - collection.key, - ); + await cryptoWorker.encryptMetadataJSON({ + jsonValue: updatedPublicMagicMetadata.data, + keyB64: collection.key, + }); const reqBody: UpdateMagicMetadataRequest = { id: collection.id, @@ -932,7 +932,7 @@ export const renameCollection = async ( await changeCollectionSubType(collection, SUB_TYPE.DEFAULT); } const token = getToken(); - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const { encryptedData: encryptedName, nonce: nameDecryptionNonce } = await cryptoWorker.encryptUTF8(newCollectionName, collection.key); const collectionRenameRequest = { @@ -956,7 +956,7 @@ export const shareCollection = async ( role: string, ) => { try { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const token = getToken(); const publicKey: string = await getPublicKey(withUserEmail); const encryptedKey = await cryptoWorker.boxSeal( diff --git a/web/apps/photos/src/services/entityService.ts b/web/apps/photos/src/services/entityService.ts index 120058f265..b91c230d37 100644 --- a/web/apps/photos/src/services/entityService.ts +++ b/web/apps/photos/src/services/entityService.ts @@ -1,6 +1,6 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; @@ -68,7 +68,7 @@ export const getEntityKey = async (type: EntityType) => { }, ); const encryptedEntityKey: EncryptedEntityKey = resp.data; - const worker = await ComlinkCryptoWorker.getInstance(); + const worker = await sharedCryptoWorker(); const masterKey = await getActualKey(); const { encryptedKey, header, ...rest } = encryptedEntityKey; const decryptedData = await worker.decryptB64( @@ -129,12 +129,12 @@ const syncEntity = async (type: EntityType): Promise> => { return entity as unknown as Entity; } const { encryptedData, header, ...rest } = entity; - const worker = await ComlinkCryptoWorker.getInstance(); - const decryptedData = await worker.decryptMetadata( - encryptedData, - header, - entityKey.data, - ); + const worker = await sharedCryptoWorker(); + const decryptedData = await worker.decryptMetadataJSON({ + encryptedDataB64: encryptedData, + decryptionHeaderB64: header, + keyB64: entityKey.data, + }); return { ...rest, data: decryptedData, diff --git a/web/apps/photos/src/services/fileService.ts b/web/apps/photos/src/services/fileService.ts index d2d4e8c49a..f7942ceb57 100644 --- a/web/apps/photos/src/services/fileService.ts +++ b/web/apps/photos/src/services/fileService.ts @@ -1,3 +1,4 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import { getLocalFiles, setLocalFiles } from "@/new/photos/services/files"; @@ -11,7 +12,6 @@ import { import { BulkUpdateMagicMetadataRequest } from "@/new/photos/types/magicMetadata"; import { mergeMetadata } from "@/new/photos/utils/file"; import { batch } from "@/utils/array"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import HTTPService from "@ente/shared/network/HTTPService"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; import { REQUEST_BATCH_SIZE } from "constants/api"; @@ -186,16 +186,16 @@ export const updateFileMagicMetadata = async ( return; } const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] }; - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); for (const { file, updatedMagicMetadata, } of fileWithUpdatedMagicMetadataList) { const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadata( - updatedMagicMetadata.data, - file.key, - ); + await cryptoWorker.encryptMetadataJSON({ + jsonValue: updatedMagicMetadata.data, + keyB64: file.key, + }); reqBody.metadataList.push({ id: file.id, magicMetadata: { @@ -233,16 +233,16 @@ export const updateFilePublicMagicMetadata = async ( return; } const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] }; - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); for (const { file, updatedPublicMagicMetadata, } of fileWithUpdatedPublicMagicMetadataList) { const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadata( - updatedPublicMagicMetadata.data, - file.key, - ); + await cryptoWorker.encryptMetadataJSON({ + jsonValue: updatedPublicMagicMetadata.data, + keyB64: file.key, + }); reqBody.metadataList.push({ id: file.id, magicMetadata: { diff --git a/web/apps/photos/src/services/publicCollectionService.ts b/web/apps/photos/src/services/publicCollectionService.ts index 9c1bed78df..0246f2cfcd 100644 --- a/web/apps/photos/src/services/publicCollectionService.ts +++ b/web/apps/photos/src/services/publicCollectionService.ts @@ -1,8 +1,8 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import { EncryptedEnteFile, EnteFile } from "@/new/photos/types/file"; import { mergeMetadata } from "@/new/photos/utils/file"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { CustomError, parseSharingErrorCodes } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import localForage from "@ente/shared/storage/localForage"; @@ -315,7 +315,7 @@ export const getPublicCollection = async ( const fetchedCollection = resp.data.collection; const referralCode = resp.data.referralCode ?? ""; - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const collectionName = (fetchedCollection.name = fetchedCollection.name || @@ -329,11 +329,12 @@ export const getPublicCollection = async ( if (fetchedCollection.pubMagicMetadata?.data) { collectionPublicMagicMetadata = { ...fetchedCollection.pubMagicMetadata, - data: await cryptoWorker.decryptMetadata( - fetchedCollection.pubMagicMetadata.data, - fetchedCollection.pubMagicMetadata.header, - collectionKey, - ), + data: await cryptoWorker.decryptMetadataJSON({ + encryptedDataB64: fetchedCollection.pubMagicMetadata.data, + decryptionHeaderB64: + fetchedCollection.pubMagicMetadata.header, + keyB64: collectionKey, + }), }; } diff --git a/web/apps/photos/src/utils/crypto/index.ts b/web/apps/photos/src/utils/crypto/index.ts index 21a296110e..11410fc512 100644 --- a/web/apps/photos/src/utils/crypto/index.ts +++ b/web/apps/photos/src/utils/crypto/index.ts @@ -1,11 +1,11 @@ -import ComlinkCryptoWorker from "@ente/shared/crypto"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { getData, LS_KEYS } from "@ente/shared/storage/localStorage"; import { getActualKey } from "@ente/shared/user"; export async function decryptDeleteAccountChallenge( encryptedChallenge: string, ) { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const masterKey = await getActualKey(); const keyAttributes = getData(LS_KEYS.KEY_ATTRIBUTES); const secretKey = await cryptoWorker.decryptB64( diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 6857585aa9..26fca50be7 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -1,3 +1,4 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { type Electron } from "@/base/types/ipc"; import { ItemVisibility } from "@/media/file-metadata"; @@ -19,7 +20,6 @@ import { mergeMetadata } from "@/new/photos/utils/file"; import { safeFileName } from "@/new/photos/utils/native-fs"; import { writeStream } from "@/new/photos/utils/native-stream"; import { withTimeout } from "@/utils/promise"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { LS_KEYS, getData } from "@ente/shared/storage/localStorage"; import type { User } from "@ente/shared/user/types"; import { downloadUsingAnchor } from "@ente/shared/utils"; @@ -133,7 +133,7 @@ export async function decryptFile( collectionKey: string, ): Promise { try { - const worker = await ComlinkCryptoWorker.getInstance(); + const worker = await sharedCryptoWorker(); const { encryptedKey, keyDecryptionNonce, @@ -147,31 +147,31 @@ export async function decryptFile( keyDecryptionNonce, collectionKey, ); - const fileMetadata = await worker.decryptMetadata( - metadata.encryptedData, - metadata.decryptionHeader, - fileKey, - ); + const fileMetadata = await worker.decryptMetadataJSON({ + encryptedDataB64: metadata.encryptedData, + decryptionHeaderB64: metadata.decryptionHeader, + keyB64: fileKey, + }); let fileMagicMetadata: FileMagicMetadata; let filePubMagicMetadata: FilePublicMagicMetadata; if (magicMetadata?.data) { fileMagicMetadata = { ...file.magicMetadata, - data: await worker.decryptMetadata( - magicMetadata.data, - magicMetadata.header, - fileKey, - ), + data: await worker.decryptMetadataJSON({ + encryptedDataB64: magicMetadata.data, + decryptionHeaderB64: magicMetadata.header, + keyB64: fileKey, + }), }; } if (pubMagicMetadata?.data) { filePubMagicMetadata = { ...pubMagicMetadata, - data: await worker.decryptMetadata( - pubMagicMetadata.data, - pubMagicMetadata.header, - fileKey, - ), + data: await worker.decryptMetadataJSON({ + encryptedDataB64: pubMagicMetadata.data, + decryptionHeaderB64: pubMagicMetadata.header, + keyB64: fileKey, + }), }; } return { diff --git a/web/apps/photos/src/utils/magicMetadata/index.ts b/web/apps/photos/src/utils/magicMetadata/index.ts index 2d80b486d2..e405836a35 100644 --- a/web/apps/photos/src/utils/magicMetadata/index.ts +++ b/web/apps/photos/src/utils/magicMetadata/index.ts @@ -1,7 +1,7 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { ItemVisibility } from "@/media/file-metadata"; import { EnteFile } from "@/new/photos/types/file"; import { MagicMetadataCore } from "@/new/photos/types/magicMetadata"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { Collection } from "types/collection"; export function isArchivedFile(item: EnteFile): boolean { @@ -46,7 +46,7 @@ export async function updateMagicMetadata( originalMagicMetadata?: MagicMetadataCore, decryptionKey?: string, ): Promise> { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); if (!originalMagicMetadata) { originalMagicMetadata = getNewMagicMetadata(); diff --git a/web/packages/accounts/pages/change-password.tsx b/web/packages/accounts/pages/change-password.tsx index a525d6c0be..113e74d89e 100644 --- a/web/packages/accounts/pages/change-password.tsx +++ b/web/packages/accounts/pages/change-password.tsx @@ -19,7 +19,6 @@ import FormPaper from "@ente/shared/components/Form/FormPaper"; import FormPaperFooter from "@ente/shared/components/Form/FormPaper/Footer"; import FormPaperTitle from "@ente/shared/components/Form/FormPaper/Title"; import LinkButton from "@ente/shared/components/LinkButton"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { generateAndSaveIntermediateKeyAttributes, generateLoginSubKey, @@ -35,6 +34,7 @@ import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { appHomeRoute } from "../services/redirect"; import type { PageProps } from "../types/page"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; const Page: React.FC = () => { const [token, setToken] = useState(); @@ -57,7 +57,7 @@ const Page: React.FC = () => { passphrase, setFieldError, ) => { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const key = await getActualKey(); const keyAttributes: KeyAttributes = getData(LS_KEYS.KEY_ATTRIBUTES); const kekSalt = await cryptoWorker.generateSaltToDeriveKey(); diff --git a/web/packages/accounts/pages/credentials.tsx b/web/packages/accounts/pages/credentials.tsx index 57b8597fec..02bf8e4054 100644 --- a/web/packages/accounts/pages/credentials.tsx +++ b/web/packages/accounts/pages/credentials.tsx @@ -1,3 +1,5 @@ +import type { B64EncryptionResult } from "@/base/crypto/libsodium"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; @@ -13,14 +15,12 @@ import { import VerifyMasterPasswordForm, { type VerifyMasterPasswordFormProps, } from "@ente/shared/components/VerifyMasterPasswordForm"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { decryptAndStoreToken, generateAndSaveIntermediateKeyAttributes, generateLoginSubKey, saveKeyInSessionStore, } from "@ente/shared/crypto/helpers"; -import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; import { CustomError } from "@ente/shared/error"; import InMemoryStore, { MS_KEYS } from "@ente/shared/storage/InMemoryStore"; import { @@ -158,7 +158,7 @@ const Page: React.FC = ({ appContext }) => { if (kekEncryptedAttributes && keyAttributes) { removeKey(SESSION_KEYS.KEY_ENCRYPTION_KEY); - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const kek = await cryptoWorker.decryptB64( kekEncryptedAttributes.encryptedData, kekEncryptedAttributes.nonce, @@ -207,7 +207,7 @@ const Page: React.FC = ({ appContext }) => { // before we let the user in. if (sessionValidityCheck) await sessionValidityCheck; - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const { keyAttributes, encryptedToken, diff --git a/web/packages/accounts/pages/recover.tsx b/web/packages/accounts/pages/recover.tsx index 497afa170e..0160cd47cd 100644 --- a/web/packages/accounts/pages/recover.tsx +++ b/web/packages/accounts/pages/recover.tsx @@ -10,7 +10,6 @@ import LinkButton from "@ente/shared/components/LinkButton"; import SingleInputForm, { type SingleInputFormProps, } from "@ente/shared/components/SingleInputForm"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { decryptAndStoreToken, saveKeyInSessionStore, @@ -24,6 +23,7 @@ import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { appHomeRoute } from "../services/redirect"; import type { PageProps } from "../types/page"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; const bip39 = require("bip39"); // mobile client library only supports english. @@ -80,7 +80,7 @@ const Page: React.FC = ({ appContext }) => { } recoveryKey = bip39.mnemonicToEntropy(recoveryKey); } - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const keyAttr = ensure(keyAttributes); const masterKey = await cryptoWorker.decryptB64( keyAttr.masterKeyEncryptedWithRecoveryKey, diff --git a/web/packages/accounts/pages/two-factor/recover.tsx b/web/packages/accounts/pages/two-factor/recover.tsx index 9eb1cf2744..b9723bf0d9 100644 --- a/web/packages/accounts/pages/two-factor/recover.tsx +++ b/web/packages/accounts/pages/two-factor/recover.tsx @@ -5,6 +5,8 @@ import { } from "@/accounts/api/user"; import { PAGES } from "@/accounts/constants/pages"; import type { AccountsContextT } from "@/accounts/types/context"; +import type { B64EncryptionResult } from "@/base/crypto/libsodium"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; @@ -16,8 +18,6 @@ import LinkButton from "@ente/shared/components/LinkButton"; import SingleInputForm, { type SingleInputFormProps, } from "@ente/shared/components/SingleInputForm"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; -import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; import { ApiError } from "@ente/shared/error"; import { LS_KEYS, @@ -117,7 +117,7 @@ const Page: React.FC = ({ appContext, twoFactorType }) => { } recoveryKey = bip39.mnemonicToEntropy(recoveryKey); } - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const { encryptedData, nonce } = ensure(encryptedTwoFactorSecret); const twoFactorSecret = await cryptoWorker.decryptB64( encryptedData, diff --git a/web/packages/accounts/services/passkey.ts b/web/packages/accounts/services/passkey.ts index 2fd605b551..5da030fb54 100644 --- a/web/packages/accounts/services/passkey.ts +++ b/web/packages/accounts/services/passkey.ts @@ -1,15 +1,12 @@ import { clientPackageName, isDesktop } from "@/base/app"; +import { encryptToB64, generateEncryptionKey } from "@/base/crypto/libsodium"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { clientPackageHeader, HTTPError } from "@/base/http"; import log from "@/base/log"; import { accountsAppOrigin, apiURL } from "@/base/origins"; import { TwoFactorAuthorizationResponse } from "@/base/types/credentials"; import { ensure } from "@/utils/ensure"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { getRecoveryKey } from "@ente/shared/crypto/helpers"; -import { - encryptToB64, - generateEncryptionKey, -} from "@ente/shared/crypto/internal/libsodium"; import { CustomError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import InMemoryStore, { MS_KEYS } from "@ente/shared/storage/InMemoryStore"; @@ -110,7 +107,7 @@ export const openAccountsManagePasskeysPage = async () => { const resetSecret = await generateEncryptionKey(); - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const encryptionResult = await encryptToB64( resetSecret, await cryptoWorker.fromHex(recoveryKey), diff --git a/web/packages/accounts/services/srp.ts b/web/packages/accounts/services/srp.ts index 9ac814de82..044a045ce9 100644 --- a/web/packages/accounts/services/srp.ts +++ b/web/packages/accounts/services/srp.ts @@ -1,6 +1,6 @@ import type { UserVerificationResponse } from "@/accounts/types/user"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { generateLoginSubKey } from "@ente/shared/crypto/helpers"; import InMemoryStore, { MS_KEYS } from "@ente/shared/storage/InMemoryStore"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; @@ -69,7 +69,7 @@ export const configureSRP = async ({ export const generateSRPSetupAttributes = async ( loginSubKey: string, ): Promise => { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const srpSalt = await cryptoWorker.generateSaltToDeriveKey(); diff --git a/web/packages/accounts/utils/srp.ts b/web/packages/accounts/utils/srp.ts index 798af9b878..248e22314b 100644 --- a/web/packages/accounts/utils/srp.ts +++ b/web/packages/accounts/utils/srp.ts @@ -1,15 +1,15 @@ -import ComlinkCryptoWorker from "@ente/shared/crypto"; import { generateLoginSubKey } from "@ente/shared/crypto/helpers"; import type { KeyAttributes } from "@ente/shared/user/types"; import { generateSRPSetupAttributes } from "../services/srp"; import type { SRPSetupAttributes } from "../types/srp"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; export async function generateKeyAndSRPAttributes(passphrase: string): Promise<{ keyAttributes: KeyAttributes; masterKey: string; srpSetupAttributes: SRPSetupAttributes; }> { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const masterKey = await cryptoWorker.generateEncryptionKey(); const recoveryKey = await cryptoWorker.generateEncryptionKey(); const kekSalt = await cryptoWorker.generateSaltToDeriveKey(); diff --git a/web/packages/new/photos/services/download.ts b/web/packages/new/photos/services/download.ts index 10b7f200bc..9b54a10bae 100644 --- a/web/packages/new/photos/services/download.ts +++ b/web/packages/new/photos/services/download.ts @@ -3,6 +3,7 @@ import { isDesktop } from "@/base/app"; import { blobCache, type BlobCache } from "@/base/blob-cache"; +import { type CryptoWorker, sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { customAPIOrigin } from "@/base/origins"; import { FileType } from "@/media/file-type"; @@ -15,13 +16,10 @@ import type { } from "@/new/photos/types/file"; import { renderableImageBlob } from "@/new/photos/utils/file"; import { ensure } from "@/utils/ensure"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; -import { DedicatedCryptoWorker } from "@ente/shared/crypto/internal/crypto.worker"; import { CustomError } from "@ente/shared/error"; import { isPlaybackPossible } from "@ente/shared/media/video-playback"; import HTTPService from "@ente/shared/network/HTTPService"; import { retryAsyncFunction } from "@ente/shared/utils"; -import type { Remote } from "comlink"; export type OnDownloadProgress = (event: { loaded: number; @@ -52,7 +50,7 @@ class DownloadManagerImpl { * Only available when we're running in the desktop app. */ private fileCache?: BlobCache; - private cryptoWorker: Remote | undefined; + private cryptoWorker: CryptoWorker | undefined; private fileObjectURLPromises = new Map>(); private fileConversionPromises = new Map>(); @@ -85,7 +83,7 @@ class DownloadManagerImpl { // } catch (e) { // log.error("Failed to open file cache, will continue without it", e); // } - this.cryptoWorker = await ComlinkCryptoWorker.getInstance(); + this.cryptoWorker = await sharedCryptoWorker(); this.ready = true; } @@ -125,11 +123,11 @@ class DownloadManagerImpl { const { downloadClient, cryptoWorker } = this.ensureInitialized(); const encrypted = await downloadClient.downloadThumbnail(file); - const decrypted = await cryptoWorker.decryptThumbnail( - encrypted, - file.thumbnail.decryptionHeader, - file.key, - ); + const decrypted = await cryptoWorker.decryptThumbnail({ + encryptedData: encrypted, + decryptionHeaderB64: file.thumbnail.decryptionHeader, + keyB64: file.key, + }); return decrypted; }; diff --git a/web/packages/shared/components/VerifyMasterPasswordForm.tsx b/web/packages/shared/components/VerifyMasterPasswordForm.tsx index a1849d8f7e..c2f962c989 100644 --- a/web/packages/shared/components/VerifyMasterPasswordForm.tsx +++ b/web/packages/shared/components/VerifyMasterPasswordForm.tsx @@ -1,11 +1,11 @@ import type { SRPAttributes } from "@/accounts/types/srp"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { Input, type ButtonProps } from "@mui/material"; import { t } from "i18next"; import SingleInputForm, { type SingleInputFormProps, } from "../components/SingleInputForm"; -import ComlinkCryptoWorker from "../crypto"; import { CustomError } from "../error"; import type { KeyAttributes, User } from "../user/types"; @@ -45,7 +45,7 @@ export default function VerifyMasterPasswordForm({ setFieldError, ) => { try { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); let kek: string; try { if (srpAttributes) { diff --git a/web/packages/shared/crypto/helpers.ts b/web/packages/shared/crypto/helpers.ts index 6c1d944d1e..6b0acb7a35 100644 --- a/web/packages/shared/crypto/helpers.ts +++ b/web/packages/shared/crypto/helpers.ts @@ -1,4 +1,5 @@ import { setRecoveryKey } from "@/accounts/api/user"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { LS_KEYS, @@ -10,7 +11,6 @@ import { getToken } from "@ente/shared/storage/localStorage/helpers"; import { SESSION_KEYS, setKey } from "@ente/shared/storage/sessionStorage"; import { getActualKey } from "@ente/shared/user"; import type { KeyAttributes } from "@ente/shared/user/types"; -import ComlinkCryptoWorker from "."; const LOGIN_SUB_KEY_LENGTH = 32; const LOGIN_SUB_KEY_ID = 1; @@ -21,7 +21,7 @@ export async function decryptAndStoreToken( keyAttributes: KeyAttributes, masterKey: string, ) { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const user = getData(LS_KEYS.USER); let decryptedToken = null; const { encryptedToken } = user; @@ -57,7 +57,7 @@ export async function generateAndSaveIntermediateKeyAttributes( existingKeyAttributes: KeyAttributes, key: string, ): Promise { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const intermediateKekSalt = await cryptoWorker.generateSaltToDeriveKey(); const intermediateKek = await cryptoWorker.deriveInteractiveKey( passphrase, @@ -80,7 +80,7 @@ export async function generateAndSaveIntermediateKeyAttributes( } export const generateLoginSubKey = async (kek: string) => { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const kekSubKeyString = await cryptoWorker.generateSubKey( kek, LOGIN_SUB_KEY_LENGTH, @@ -102,7 +102,7 @@ export const saveKeyInSessionStore = async ( key: string, fromDesktop?: boolean, ) => { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const sessionKeyAttributes = await cryptoWorker.generateKeyAndEncryptToB64(key); setKey(keyType, sessionKeyAttributes); @@ -113,7 +113,7 @@ export const saveKeyInSessionStore = async ( }; export async function encryptWithRecoveryKey(key: string) { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const hexRecoveryKey = await getRecoveryKey(); const recoveryKey = await cryptoWorker.fromHex(hexRecoveryKey); const encryptedKey = await cryptoWorker.encryptToB64(key, recoveryKey); @@ -122,7 +122,7 @@ export async function encryptWithRecoveryKey(key: string) { export const getRecoveryKey = async () => { try { - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const keyAttributes: KeyAttributes = getData(LS_KEYS.KEY_ATTRIBUTES); const { @@ -155,7 +155,7 @@ async function createNewRecoveryKey() { const masterKey = await getActualKey(); const existingAttributes = getData(LS_KEYS.KEY_ATTRIBUTES); - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const recoveryKey = await cryptoWorker.generateEncryptionKey(); const encryptedMasterKey = await cryptoWorker.encryptToB64( diff --git a/web/packages/shared/file-metadata.ts b/web/packages/shared/file-metadata.ts index 869cc9db6b..71ee155ab1 100644 --- a/web/packages/shared/file-metadata.ts +++ b/web/packages/shared/file-metadata.ts @@ -1,4 +1,5 @@ import { decryptMetadata } from "@/base/crypto/ente"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { isDevBuild } from "@/base/env"; import { decryptPublicMagicMetadata, @@ -6,7 +7,6 @@ import { } from "@/media/file-metadata"; import { EnteFile } from "@/new/photos/types/file"; import { fileLogID } from "@/new/photos/utils/file"; -import ComlinkCryptoWorker from "@ente/shared/crypto"; /** * On-demand decrypt the public magic metadata for an {@link EnteFile} for code @@ -18,7 +18,7 @@ import ComlinkCryptoWorker from "@ente/shared/crypto"; export const getPublicMagicMetadataMT = async (enteFile: EnteFile) => decryptPublicMagicMetadata( enteFile, - (await ComlinkCryptoWorker.getInstance()).decryptMetadata, + (await sharedCryptoWorker()).decryptMetadata, ); /** diff --git a/web/packages/shared/user/index.ts b/web/packages/shared/user/index.ts index f66a62b4f6..69a9780b47 100644 --- a/web/packages/shared/user/index.ts +++ b/web/packages/shared/user/index.ts @@ -1,5 +1,5 @@ -import ComlinkCryptoWorker from "@ente/shared/crypto"; -import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; +import type { B64EncryptionResult } from "@/base/crypto/libsodium"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { CustomError } from "@ente/shared/error"; import { getKey, SESSION_KEYS } from "@ente/shared/storage/sessionStorage"; @@ -9,7 +9,7 @@ export const getActualKey = async () => { SESSION_KEYS.ENCRYPTION_KEY, ); - const cryptoWorker = await ComlinkCryptoWorker.getInstance(); + const cryptoWorker = await sharedCryptoWorker(); const key = await cryptoWorker.decryptB64( encryptionKeyAttributes.encryptedData, encryptionKeyAttributes.nonce, From b8830144be7da4f269dbec62547ebf6388025b54 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 19:54:11 +0530 Subject: [PATCH 159/211] Remove the no longer needed indirection --- web/packages/media/file-metadata.ts | 55 +++++----------------------- web/packages/shared/file-metadata.ts | 17 +-------- 2 files changed, 11 insertions(+), 61 deletions(-) diff --git a/web/packages/media/file-metadata.ts b/web/packages/media/file-metadata.ts index 3e75dac66f..18fbabf1b3 100644 --- a/web/packages/media/file-metadata.ts +++ b/web/packages/media/file-metadata.ts @@ -1,4 +1,4 @@ -import { encryptMetadata, type decryptMetadata } from "@/base/crypto/ente"; +import { decryptMetadataJSON, encryptMetadataJSON } from "@/base/crypto/ente"; import { authenticatedRequestHeaders, ensureOk } from "@/base/http"; import { apiURL } from "@/base/origins"; import { type EnteFile } from "@/new/photos/types/file"; @@ -270,26 +270,6 @@ const PublicMagicMetadata = z }) .passthrough(); -/** - * A function that can be used to encrypt the contents of a metadata field - * associated with a file. - * - * This is parameterized to allow us to use either the regular - * {@link encryptMetadata} (if we're already running in a web worker) or its web - * worker wrapper (if we're running on the main thread). - */ -export type EncryptMetadataF = typeof encryptMetadata; - -/** - * A function that can be used to decrypt the contents of a metadata field - * associated with a file. - * - * This is parameterized to allow us to use either the regular - * {@link encryptMetadata} (if we're already running in a web worker) or its web - * worker wrapper (if we're running on the main thread). - */ -export type DecryptMetadataF = typeof decryptMetadata; - /** * Return the public magic metadata for the given {@link enteFile}. * @@ -302,7 +282,6 @@ export type DecryptMetadataF = typeof decryptMetadata; */ export const decryptPublicMagicMetadata = async ( enteFile: EnteFile, - decryptMetadataF: DecryptMetadataF, ): Promise => { const envelope = enteFile.pubMagicMetadata; // TODO: The underlying types need auditing. @@ -317,11 +296,11 @@ export const decryptPublicMagicMetadata = async ( const jsonValue = typeof envelope.data == "string" - ? await decryptMetadataF( - envelope.data, - envelope.header, - enteFile.key, - ) + ? await decryptMetadataJSON({ + encryptedDataB64: envelope.data, + decryptionHeaderB64: envelope.header, + keyB64: enteFile.key, + }) : envelope.data; const result = PublicMagicMetadata.parse( // TODO: Can we avoid this cast? @@ -377,23 +356,12 @@ export const getUICreationDate = ( * * @param metadataUpdates A subset of {@link PublicMagicMetadata} containing the * fields that we want to add or update. - * - * @param encryptMetadataF A function that is used to encrypt the updated - * metadata. - * - * @param decryptMetadataF A function that is used to decrypt the existing - * metadata. */ export const updateRemotePublicMagicMetadata = async ( enteFile: EnteFile, metadataUpdates: Partial, - encryptMetadataF: EncryptMetadataF, - decryptMetadataF: DecryptMetadataF, ) => { - const existingMetadata = await decryptPublicMagicMetadata( - enteFile, - decryptMetadataF, - ); + const existingMetadata = await decryptPublicMagicMetadata(enteFile); const updatedMetadata = { ...(existingMetadata ?? {}), ...metadataUpdates }; @@ -405,7 +373,6 @@ export const updateRemotePublicMagicMetadata = async ( enteFile, updatedMetadata, metadataVersion, - encryptMetadataF, ); const updatedEnvelope = ensure(updateRequest.metadataList[0]).magicMetadata; @@ -420,7 +387,7 @@ export const updateRemotePublicMagicMetadata = async ( // response of the /diff. Temporarily bump it for the in place edits. enteFile.pubMagicMetadata.version = enteFile.pubMagicMetadata.version + 1; // Re-read the data. - await decryptPublicMagicMetadata(enteFile, decryptMetadataF); + await decryptPublicMagicMetadata(enteFile); // Re-jig the other bits of EnteFile that depend on its public magic // metadata. mergeMetadata1(enteFile); @@ -492,7 +459,6 @@ const updateMagicMetadataRequest = async ( enteFile: EnteFile, metadata: PrivateMagicMetadata | PublicMagicMetadata, metadataVersion: number, - encryptMetadataF: EncryptMetadataF, ): Promise => { // Drop all null or undefined values to obtain the syncable entries. // See: [Note: Optional magic metadata keys]. @@ -500,9 +466,8 @@ const updateMagicMetadataRequest = async ( ([, v]) => v !== null && v !== undefined, ); - const { encryptedDataB64, decryptionHeaderB64 } = await encryptMetadataF( - Object.fromEntries(validEntries), - enteFile.key, + const { encryptedDataB64, decryptionHeaderB64 } = await encryptMetadataJSON( + { jsonValue: Object.fromEntries(validEntries), keyB64: enteFile.key }, ); return { diff --git a/web/packages/shared/file-metadata.ts b/web/packages/shared/file-metadata.ts index 71ee155ab1..a5608ed382 100644 --- a/web/packages/shared/file-metadata.ts +++ b/web/packages/shared/file-metadata.ts @@ -1,5 +1,3 @@ -import { decryptMetadata } from "@/base/crypto/ente"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; import { isDevBuild } from "@/base/env"; import { decryptPublicMagicMetadata, @@ -8,19 +6,6 @@ import { import { EnteFile } from "@/new/photos/types/file"; import { fileLogID } from "@/new/photos/utils/file"; -/** - * On-demand decrypt the public magic metadata for an {@link EnteFile} for code - * running on the main thread. - * - * It both modifies the given file object, and also returns the decrypted - * metadata. - */ -export const getPublicMagicMetadataMT = async (enteFile: EnteFile) => - decryptPublicMagicMetadata( - enteFile, - (await sharedCryptoWorker()).decryptMetadata, - ); - /** * On-demand decrypt the public magic metadata for an {@link EnteFile} for code * running on the main thread, but do it synchronously. @@ -40,7 +25,7 @@ export const getPublicMagicMetadataMTSync = (enteFile: EnteFile) => { throw new Error( `Public magic metadata for ${fileLogID(enteFile)} had not been decrypted even when the file reached the UI layer`, ); - decryptPublicMagicMetadata(enteFile, decryptMetadata); + decryptPublicMagicMetadata(enteFile); } // This cast is unavoidable in the current setup. We need to refactor the // types so that this cast in not needed. From b7de8ca9e5bd6c462d25c7029b6dfb514c44d57b Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 19:57:12 +0530 Subject: [PATCH 160/211] Update import --- web/apps/accounts/src/services/passkey.ts | 10 +++++----- web/apps/cast/src/services/pair.ts | 5 +---- .../Collections/CollectionOptions/AlbumCastDialog.tsx | 2 +- web/packages/accounts/api/user.ts | 2 +- web/packages/accounts/pages/passkeys/finish.tsx | 2 +- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/web/apps/accounts/src/services/passkey.ts b/web/apps/accounts/src/services/passkey.ts index f470b1d1cc..3c12e31d79 100644 --- a/web/apps/accounts/src/services/passkey.ts +++ b/web/apps/accounts/src/services/passkey.ts @@ -1,15 +1,15 @@ import { clientPackageName } from "@/base/app"; +import { + fromB64URLSafeNoPadding, + toB64URLSafeNoPadding, + toB64URLSafeNoPaddingString, +} from "@/base/crypto/libsodium"; import { isDevBuild } from "@/base/env"; import { clientPackageHeader, ensureOk, HTTPError } from "@/base/http"; import { apiURL } from "@/base/origins"; import { TwoFactorAuthorizationResponse } from "@/base/types/credentials"; import { ensure } from "@/utils/ensure"; import { nullToUndefined } from "@/utils/transform"; -import { - fromB64URLSafeNoPadding, - toB64URLSafeNoPadding, - toB64URLSafeNoPaddingString, -} from "@ente/shared/crypto/internal/libsodium"; import { z } from "zod"; /** Return true if the user's browser supports WebAuthn (Passkeys). */ diff --git a/web/apps/cast/src/services/pair.ts b/web/apps/cast/src/services/pair.ts index 287122c456..378165eb2b 100644 --- a/web/apps/cast/src/services/pair.ts +++ b/web/apps/cast/src/services/pair.ts @@ -1,9 +1,6 @@ +import { boxSealOpen, generateKeyPair } from "@/base/crypto/libsodium"; import log from "@/base/log"; import { wait } from "@/utils/promise"; -import { - boxSealOpen, - generateKeyPair, -} from "@ente/shared/crypto/internal/libsodium"; import castGateway from "@ente/shared/network/cast"; export interface Registration { diff --git a/web/apps/photos/src/components/Collections/CollectionOptions/AlbumCastDialog.tsx b/web/apps/photos/src/components/Collections/CollectionOptions/AlbumCastDialog.tsx index f9d83bd099..75ec3b4626 100644 --- a/web/apps/photos/src/components/Collections/CollectionOptions/AlbumCastDialog.tsx +++ b/web/apps/photos/src/components/Collections/CollectionOptions/AlbumCastDialog.tsx @@ -1,3 +1,4 @@ +import { boxSeal } from "@/base/crypto/libsodium"; import log from "@/base/log"; import { VerticallyCentered } from "@ente/shared/components/Container"; import DialogBoxV2 from "@ente/shared/components/DialogBoxV2"; @@ -6,7 +7,6 @@ import EnteSpinner from "@ente/shared/components/EnteSpinner"; import SingleInputForm, { type SingleInputFormProps, } from "@ente/shared/components/SingleInputForm"; -import { boxSeal } from "@ente/shared/crypto/internal/libsodium"; import castGateway from "@ente/shared/network/cast"; import { Link, Typography } from "@mui/material"; import { t } from "i18next"; diff --git a/web/packages/accounts/api/user.ts b/web/packages/accounts/api/user.ts index 9e22a5ab68..667eaf10c9 100644 --- a/web/packages/accounts/api/user.ts +++ b/web/packages/accounts/api/user.ts @@ -6,8 +6,8 @@ import type { UserVerificationResponse, } from "@/accounts/types/user"; import { appName } from "@/base/app"; +import type { B64EncryptionResult } from "@/base/crypto/libsodium"; import { apiURL } from "@/base/origins"; -import type { B64EncryptionResult } from "@ente/shared/crypto/internal/libsodium"; import { ApiError, CustomError } from "@ente/shared/error"; import HTTPService from "@ente/shared/network/HTTPService"; import { getToken } from "@ente/shared/storage/localStorage/helpers"; diff --git a/web/packages/accounts/pages/passkeys/finish.tsx b/web/packages/accounts/pages/passkeys/finish.tsx index 1b73aca1d7..04c17803c1 100644 --- a/web/packages/accounts/pages/passkeys/finish.tsx +++ b/web/packages/accounts/pages/passkeys/finish.tsx @@ -1,8 +1,8 @@ +import { fromB64URLSafeNoPaddingString } from "@/base/crypto/libsodium"; import log from "@/base/log"; import { nullToUndefined } from "@/utils/transform"; import { VerticallyCentered } from "@ente/shared/components/Container"; import EnteSpinner from "@ente/shared/components/EnteSpinner"; -import { fromB64URLSafeNoPaddingString } from "@ente/shared/crypto/internal/libsodium"; import InMemoryStore, { MS_KEYS } from "@ente/shared/storage/InMemoryStore"; import { LS_KEYS, From de032656754967a717150930b0ddcf16e156f374 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 19:58:22 +0530 Subject: [PATCH 161/211] Leftovers --- .../photos/src/components/FixCreationTime.tsx | 23 +++++-------------- .../components/PhotoViewer/FileInfo/index.tsx | 12 ++++------ .../accounts/pages/change-password.tsx | 2 +- web/packages/accounts/pages/recover.tsx | 2 +- web/packages/accounts/utils/srp.ts | 2 +- .../new/photos/services/ml/embedding.ts | 12 +++++----- 6 files changed, 19 insertions(+), 34 deletions(-) diff --git a/web/apps/photos/src/components/FixCreationTime.tsx b/web/apps/photos/src/components/FixCreationTime.tsx index b5e3100ccc..d427b52ad5 100644 --- a/web/apps/photos/src/components/FixCreationTime.tsx +++ b/web/apps/photos/src/components/FixCreationTime.tsx @@ -1,4 +1,3 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { decryptPublicMagicMetadata, @@ -353,25 +352,15 @@ const updateEnteFileDate = async ( if (!newDate) return; - const cryptoWorker = await sharedCryptoWorker(); - const existingUIDate = getUICreationDate( enteFile, - await decryptPublicMagicMetadata( - enteFile, - cryptoWorker.decryptMetadata, - ), + await decryptPublicMagicMetadata(enteFile), ); if (newDate.timestamp == existingUIDate.getTime()) return; - await updateRemotePublicMagicMetadata( - enteFile, - { - dateTime: newDate.dateTime, - offsetTime: newDate.offset, - editedTime: newDate.timestamp, - }, - cryptoWorker.encryptMetadata, - cryptoWorker.decryptMetadata, - ); + await updateRemotePublicMagicMetadata(enteFile, { + dateTime: newDate.dateTime, + offsetTime: newDate.offset, + editedTime: newDate.timestamp, + }); }; diff --git a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx index 896a82facf..90e6c1e027 100644 --- a/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx +++ b/web/apps/photos/src/components/PhotoViewer/FileInfo/index.tsx @@ -1,6 +1,5 @@ import { EnteDrawer } from "@/base/components/EnteDrawer"; import { Titlebar } from "@/base/components/Titlebar"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; import { nameAndExtension } from "@/base/file"; import log from "@/base/log"; import type { ParsedMetadata } from "@/media/file-metadata"; @@ -399,13 +398,10 @@ export const CreationTime: React.FC = ({ return; } - const cryptoWorker = await sharedCryptoWorker(); - await updateRemotePublicMagicMetadata( - enteFile, - { dateTime, editedTime: timestamp }, - cryptoWorker.encryptMetadata, - cryptoWorker.decryptMetadata, - ); + await updateRemotePublicMagicMetadata(enteFile, { + dateTime, + editedTime: timestamp, + }); scheduleUpdate(); } diff --git a/web/packages/accounts/pages/change-password.tsx b/web/packages/accounts/pages/change-password.tsx index 113e74d89e..9d0c2b3f77 100644 --- a/web/packages/accounts/pages/change-password.tsx +++ b/web/packages/accounts/pages/change-password.tsx @@ -13,6 +13,7 @@ import { } from "@/accounts/services/srp"; import type { UpdatedKey } from "@/accounts/types/user"; import { convertBase64ToBuffer, convertBufferToBase64 } from "@/accounts/utils"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; import FormPaper from "@ente/shared/components/Form/FormPaper"; @@ -34,7 +35,6 @@ import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { appHomeRoute } from "../services/redirect"; import type { PageProps } from "../types/page"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; const Page: React.FC = () => { const [token, setToken] = useState(); diff --git a/web/packages/accounts/pages/recover.tsx b/web/packages/accounts/pages/recover.tsx index 0160cd47cd..281f3152d2 100644 --- a/web/packages/accounts/pages/recover.tsx +++ b/web/packages/accounts/pages/recover.tsx @@ -1,5 +1,6 @@ import { sendOtt } from "@/accounts/api/user"; import { PAGES } from "@/accounts/constants/pages"; +import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; @@ -23,7 +24,6 @@ import { useRouter } from "next/router"; import { useEffect, useState } from "react"; import { appHomeRoute } from "../services/redirect"; import type { PageProps } from "../types/page"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; const bip39 = require("bip39"); // mobile client library only supports english. diff --git a/web/packages/accounts/utils/srp.ts b/web/packages/accounts/utils/srp.ts index 248e22314b..21d1fa4ad3 100644 --- a/web/packages/accounts/utils/srp.ts +++ b/web/packages/accounts/utils/srp.ts @@ -1,8 +1,8 @@ +import { sharedCryptoWorker } from "@/base/crypto/worker"; import { generateLoginSubKey } from "@ente/shared/crypto/helpers"; import type { KeyAttributes } from "@ente/shared/user/types"; import { generateSRPSetupAttributes } from "../services/srp"; import type { SRPSetupAttributes } from "../types/srp"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; export async function generateKeyAndSRPAttributes(passphrase: string): Promise<{ keyAttributes: KeyAttributes; diff --git a/web/packages/new/photos/services/ml/embedding.ts b/web/packages/new/photos/services/ml/embedding.ts index c0adb2cdb9..9d8bff1a9d 100644 --- a/web/packages/new/photos/services/ml/embedding.ts +++ b/web/packages/new/photos/services/ml/embedding.ts @@ -192,11 +192,11 @@ export const fetchDerivedData = async ( } try { - const decryptedBytes = await decryptFileEmbedding( - remoteEmbedding.encryptedEmbedding, - remoteEmbedding.decryptionHeader, - file.key, - ); + const decryptedBytes = await decryptFileEmbedding({ + encryptedDataB64: remoteEmbedding.encryptedEmbedding, + decryptionHeaderB64: remoteEmbedding.decryptionHeader, + keyB64: file.key, + }); const jsonString = await gunzip(decryptedBytes); result.set(fileID, remoteDerivedDataFromJSONString(jsonString)); } catch (e) { @@ -291,7 +291,7 @@ const putEmbedding = async ( embedding: Uint8Array, ) => { const { encryptedDataB64, decryptionHeaderB64 } = - await encryptFileEmbedding(embedding, enteFile.key); + await encryptFileEmbedding({ data: embedding, keyB64: enteFile.key }); const res = await fetch(await apiURL("/embeddings"), { method: "PUT", From 2b7ee9f42fecca030036e3d3976d44d5d42f049a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 20:03:40 +0530 Subject: [PATCH 162/211] Forward --- web/packages/base/crypto/ente.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index e4f1d64298..7a761ab0c5 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -127,7 +127,9 @@ export const encryptFileEmbedding = async (r: EncryptBytes) => * Use {@link decryptMetadataJSON} to decrypt the result. */ export const encryptMetadataJSON = async (r: EncryptJSON) => - assertInWorker(ei._encryptMetadataJSON(r)); + inWorker() + ? ei._encryptMetadataJSON(r) + : sharedCryptoWorker().then((w) => w.encryptMetadataJSON(r)); /** * Decrypt arbitrary data associated with an Ente object (file, collection or From a7fcf7da9baa77e7531b56505aa1d9d3a898c6b2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 20:04:50 +0530 Subject: [PATCH 163/211] Fixes --- web/packages/new/photos/services/user-entity.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/services/user-entity.ts b/web/packages/new/photos/services/user-entity.ts index 8641fd3ceb..3383ed42de 100644 --- a/web/packages/new/photos/services/user-entity.ts +++ b/web/packages/new/photos/services/user-entity.ts @@ -91,8 +91,12 @@ export const userEntityDiff = async ( sinceTime: number, entityKeyB64: string, ): Promise => { - const decrypt = (dataB64: string, headerB64: string) => - decryptMetadataBytes(dataB64, headerB64, entityKeyB64); + const decrypt = (encryptedDataB64: string, decryptionHeaderB64: string) => + decryptMetadataBytes({ + encryptedDataB64, + decryptionHeaderB64, + keyB64: entityKeyB64, + }); const params = new URLSearchParams({ type, From ccceb8c26a408f49d8dc24b738cbf90d34a57c46 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 20:08:40 +0530 Subject: [PATCH 164/211] Direct --- .../photos/src/services/collectionService.ts | 33 ++++++------------- web/apps/photos/src/services/fileService.ts | 8 ++--- 2 files changed, 13 insertions(+), 28 deletions(-) diff --git a/web/apps/photos/src/services/collectionService.ts b/web/apps/photos/src/services/collectionService.ts index 9a25b8a9cb..8fbf668453 100644 --- a/web/apps/photos/src/services/collectionService.ts +++ b/web/apps/photos/src/services/collectionService.ts @@ -1,3 +1,4 @@ +import { encryptMetadataJSON } from "@/base/crypto/ente"; import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; @@ -797,13 +798,9 @@ export const updateCollectionMagicMetadata = async ( return; } - const cryptoWorker = await sharedCryptoWorker(); - - const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadataJSON({ - jsonValue: updatedMagicMetadata.data, - keyB64: collection.key, - }); + const { encryptedDataB64, decryptionHeaderB64 } = await encryptMetadataJSON( + { jsonValue: updatedMagicMetadata.data, keyB64: collection.key }, + ); const reqBody: UpdateMagicMetadataRequest = { id: collection.id, @@ -842,14 +839,9 @@ export const updateSharedCollectionMagicMetadata = async ( return; } - const cryptoWorker = await sharedCryptoWorker(); - - const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadataJSON({ - jsonValue: updatedMagicMetadata.data, - keyB64: collection.key, - }); - + const { encryptedDataB64, decryptionHeaderB64 } = await encryptMetadataJSON( + { jsonValue: updatedMagicMetadata.data, keyB64: collection.key }, + ); const reqBody: UpdateMagicMetadataRequest = { id: collection.id, magicMetadata: { @@ -887,14 +879,9 @@ export const updatePublicCollectionMagicMetadata = async ( return; } - const cryptoWorker = await sharedCryptoWorker(); - - const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadataJSON({ - jsonValue: updatedPublicMagicMetadata.data, - keyB64: collection.key, - }); - + const { encryptedDataB64, decryptionHeaderB64 } = await encryptMetadataJSON( + { jsonValue: updatedPublicMagicMetadata.data, keyB64: collection.key }, + ); const reqBody: UpdateMagicMetadataRequest = { id: collection.id, magicMetadata: { diff --git a/web/apps/photos/src/services/fileService.ts b/web/apps/photos/src/services/fileService.ts index f7942ceb57..5cd16129c5 100644 --- a/web/apps/photos/src/services/fileService.ts +++ b/web/apps/photos/src/services/fileService.ts @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { encryptMetadataJSON } from "@/base/crypto/ente"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import { getLocalFiles, setLocalFiles } from "@/new/photos/services/files"; @@ -186,13 +186,12 @@ export const updateFileMagicMetadata = async ( return; } const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] }; - const cryptoWorker = await sharedCryptoWorker(); for (const { file, updatedMagicMetadata, } of fileWithUpdatedMagicMetadataList) { const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadataJSON({ + await encryptMetadataJSON({ jsonValue: updatedMagicMetadata.data, keyB64: file.key, }); @@ -233,13 +232,12 @@ export const updateFilePublicMagicMetadata = async ( return; } const reqBody: BulkUpdateMagicMetadataRequest = { metadataList: [] }; - const cryptoWorker = await sharedCryptoWorker(); for (const { file, updatedPublicMagicMetadata, } of fileWithUpdatedPublicMagicMetadataList) { const { encryptedDataB64, decryptionHeaderB64 } = - await cryptoWorker.encryptMetadataJSON({ + await encryptMetadataJSON({ jsonValue: updatedPublicMagicMetadata.data, keyB64: file.key, }); From e6250e2cc30157f84ae37cc23dfbbbfe7d51424a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 20:11:29 +0530 Subject: [PATCH 165/211] Reorder --- web/packages/base/crypto/worker/worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/base/crypto/worker/worker.ts b/web/packages/base/crypto/worker/worker.ts index ad86690d02..df9b507f8c 100644 --- a/web/packages/base/crypto/worker/worker.ts +++ b/web/packages/base/crypto/worker/worker.ts @@ -15,8 +15,8 @@ export class CryptoWorker { encryptThumbnail = ei._encryptThumbnail; encryptMetadataJSON = ei._encryptMetadataJSON; decryptThumbnail = ei._decryptThumbnail; - decryptMetadataJSON = ei._decryptMetadataJSON; decryptMetadataBytes = ei._decryptMetadataBytes; + decryptMetadataJSON = ei._decryptMetadataJSON; // TODO: -- AUDIT BELOW -- From 98a9fc39ec8a7a19da1b9fbddd6b26929e4078e0 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 20:12:36 +0530 Subject: [PATCH 166/211] Direct --- web/apps/photos/src/services/entityService.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/apps/photos/src/services/entityService.ts b/web/apps/photos/src/services/entityService.ts index b91c230d37..744606f64c 100644 --- a/web/apps/photos/src/services/entityService.ts +++ b/web/apps/photos/src/services/entityService.ts @@ -1,3 +1,4 @@ +import { decryptMetadataJSON } from "@/base/crypto/ente"; import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; @@ -129,8 +130,7 @@ const syncEntity = async (type: EntityType): Promise> => { return entity as unknown as Entity; } const { encryptedData, header, ...rest } = entity; - const worker = await sharedCryptoWorker(); - const decryptedData = await worker.decryptMetadataJSON({ + const decryptedData = await decryptMetadataJSON({ encryptedDataB64: encryptedData, decryptionHeaderB64: header, keyB64: entityKey.data, From 004dd3bd0cc4e893ee45d66ccb69d5622d56cdb2 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 20:26:41 +0530 Subject: [PATCH 167/211] Rearrange to make webpack happy Something in the previous arrangement was causing webpack to not pack worker/worker.ts as a web worker. --- web/apps/auth/src/services/remote.ts | 2 +- web/apps/cast/src/services/render.ts | 2 +- .../publicShare/manage/linkPassword/setPassword.tsx | 2 +- web/apps/photos/src/pages/shared-albums/index.tsx | 2 +- web/apps/photos/src/services/collectionService.ts | 2 +- web/apps/photos/src/services/entityService.ts | 2 +- web/apps/photos/src/services/publicCollectionService.ts | 2 +- web/apps/photos/src/services/upload/uploadManager.ts | 6 ++---- web/apps/photos/src/utils/crypto/index.ts | 2 +- web/apps/photos/src/utils/file/index.ts | 2 +- web/apps/photos/src/utils/magicMetadata/index.ts | 2 +- web/packages/accounts/pages/change-password.tsx | 2 +- web/packages/accounts/pages/credentials.tsx | 2 +- web/packages/accounts/pages/recover.tsx | 2 +- web/packages/accounts/pages/two-factor/recover.tsx | 2 +- web/packages/accounts/services/passkey.ts | 2 +- web/packages/accounts/services/srp.ts | 2 +- web/packages/accounts/utils/srp.ts | 2 +- web/packages/base/crypto/ente.ts | 2 +- web/packages/base/crypto/{worker => }/index.ts | 5 ----- web/packages/base/crypto/{worker => }/worker.ts | 4 ++-- web/packages/new/photos/services/download.ts | 3 ++- web/packages/shared/components/VerifyMasterPasswordForm.tsx | 2 +- web/packages/shared/crypto/helpers.ts | 2 +- web/packages/shared/user/index.ts | 2 +- 25 files changed, 27 insertions(+), 33 deletions(-) rename web/packages/base/crypto/{worker => }/index.ts (87%) rename web/packages/base/crypto/{worker => }/worker.ts (98%) diff --git a/web/apps/auth/src/services/remote.ts b/web/apps/auth/src/services/remote.ts index e30b8d8f90..23cf643be8 100644 --- a/web/apps/auth/src/services/remote.ts +++ b/web/apps/auth/src/services/remote.ts @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import { ensureString } from "@/utils/ensure"; diff --git a/web/apps/cast/src/services/render.ts b/web/apps/cast/src/services/render.ts index 1ba9aa502b..e7bb27654c 100644 --- a/web/apps/cast/src/services/render.ts +++ b/web/apps/cast/src/services/render.ts @@ -5,7 +5,7 @@ /* eslint-disable @typescript-eslint/no-unsafe-member-access */ /* eslint-disable @typescript-eslint/no-unsafe-assignment */ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import { nameAndExtension } from "@/base/file"; import log from "@/base/log"; import { apiURL, customAPIOrigin } from "@/base/origins"; diff --git a/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/setPassword.tsx b/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/setPassword.tsx index ed33f8565a..0307d6e81f 100644 --- a/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/setPassword.tsx +++ b/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/setPassword.tsx @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import SingleInputForm, { type SingleInputFormProps, } from "@ente/shared/components/SingleInputForm"; diff --git a/web/apps/photos/src/pages/shared-albums/index.tsx b/web/apps/photos/src/pages/shared-albums/index.tsx index 64cc325660..bd87a4922a 100644 --- a/web/apps/photos/src/pages/shared-albums/index.tsx +++ b/web/apps/photos/src/pages/shared-albums/index.tsx @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import log from "@/base/log"; import downloadManager from "@/new/photos/services/download"; import { EnteFile } from "@/new/photos/types/file"; diff --git a/web/apps/photos/src/services/collectionService.ts b/web/apps/photos/src/services/collectionService.ts index 8fbf668453..0558c7473f 100644 --- a/web/apps/photos/src/services/collectionService.ts +++ b/web/apps/photos/src/services/collectionService.ts @@ -1,5 +1,5 @@ +import { sharedCryptoWorker } from "@/base/crypto"; import { encryptMetadataJSON } from "@/base/crypto/ente"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import { ItemVisibility } from "@/media/file-metadata"; diff --git a/web/apps/photos/src/services/entityService.ts b/web/apps/photos/src/services/entityService.ts index 744606f64c..2516a185db 100644 --- a/web/apps/photos/src/services/entityService.ts +++ b/web/apps/photos/src/services/entityService.ts @@ -1,5 +1,5 @@ +import { sharedCryptoWorker } from "@/base/crypto"; import { decryptMetadataJSON } from "@/base/crypto/ente"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import HTTPService from "@ente/shared/network/HTTPService"; diff --git a/web/apps/photos/src/services/publicCollectionService.ts b/web/apps/photos/src/services/publicCollectionService.ts index 0246f2cfcd..8b1a522954 100644 --- a/web/apps/photos/src/services/publicCollectionService.ts +++ b/web/apps/photos/src/services/publicCollectionService.ts @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import log from "@/base/log"; import { apiURL } from "@/base/origins"; import { EncryptedEnteFile, EnteFile } from "@/new/photos/types/file"; diff --git a/web/apps/photos/src/services/upload/uploadManager.ts b/web/apps/photos/src/services/upload/uploadManager.ts index 0ceff98c06..4b9156685e 100644 --- a/web/apps/photos/src/services/upload/uploadManager.ts +++ b/web/apps/photos/src/services/upload/uploadManager.ts @@ -1,7 +1,5 @@ -import { - createComlinkCryptoWorker, - type CryptoWorker, -} from "@/base/crypto/worker"; +import { createComlinkCryptoWorker } from "@/base/crypto"; +import { type CryptoWorker } from "@/base/crypto/worker"; import { ensureElectron } from "@/base/electron"; import { lowercaseExtension, nameAndExtension } from "@/base/file"; import log from "@/base/log"; diff --git a/web/apps/photos/src/utils/crypto/index.ts b/web/apps/photos/src/utils/crypto/index.ts index 11410fc512..52ed93f36d 100644 --- a/web/apps/photos/src/utils/crypto/index.ts +++ b/web/apps/photos/src/utils/crypto/index.ts @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import { getData, LS_KEYS } from "@ente/shared/storage/localStorage"; import { getActualKey } from "@ente/shared/user"; diff --git a/web/apps/photos/src/utils/file/index.ts b/web/apps/photos/src/utils/file/index.ts index 26fca50be7..6ca064a92b 100644 --- a/web/apps/photos/src/utils/file/index.ts +++ b/web/apps/photos/src/utils/file/index.ts @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import log from "@/base/log"; import { type Electron } from "@/base/types/ipc"; import { ItemVisibility } from "@/media/file-metadata"; diff --git a/web/apps/photos/src/utils/magicMetadata/index.ts b/web/apps/photos/src/utils/magicMetadata/index.ts index e405836a35..f07254b5df 100644 --- a/web/apps/photos/src/utils/magicMetadata/index.ts +++ b/web/apps/photos/src/utils/magicMetadata/index.ts @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import { ItemVisibility } from "@/media/file-metadata"; import { EnteFile } from "@/new/photos/types/file"; import { MagicMetadataCore } from "@/new/photos/types/magicMetadata"; diff --git a/web/packages/accounts/pages/change-password.tsx b/web/packages/accounts/pages/change-password.tsx index 9d0c2b3f77..7523d0811a 100644 --- a/web/packages/accounts/pages/change-password.tsx +++ b/web/packages/accounts/pages/change-password.tsx @@ -13,7 +13,7 @@ import { } from "@/accounts/services/srp"; import type { UpdatedKey } from "@/accounts/types/user"; import { convertBase64ToBuffer, convertBufferToBase64 } from "@/accounts/utils"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; import FormPaper from "@ente/shared/components/Form/FormPaper"; diff --git a/web/packages/accounts/pages/credentials.tsx b/web/packages/accounts/pages/credentials.tsx index 02bf8e4054..53ebb07590 100644 --- a/web/packages/accounts/pages/credentials.tsx +++ b/web/packages/accounts/pages/credentials.tsx @@ -1,5 +1,5 @@ +import { sharedCryptoWorker } from "@/base/crypto"; import type { B64EncryptionResult } from "@/base/crypto/libsodium"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; diff --git a/web/packages/accounts/pages/recover.tsx b/web/packages/accounts/pages/recover.tsx index 281f3152d2..4c39ae1eb6 100644 --- a/web/packages/accounts/pages/recover.tsx +++ b/web/packages/accounts/pages/recover.tsx @@ -1,6 +1,6 @@ import { sendOtt } from "@/accounts/api/user"; import { PAGES } from "@/accounts/constants/pages"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; diff --git a/web/packages/accounts/pages/two-factor/recover.tsx b/web/packages/accounts/pages/two-factor/recover.tsx index b9723bf0d9..2da68bbe10 100644 --- a/web/packages/accounts/pages/two-factor/recover.tsx +++ b/web/packages/accounts/pages/two-factor/recover.tsx @@ -5,8 +5,8 @@ import { } from "@/accounts/api/user"; import { PAGES } from "@/accounts/constants/pages"; import type { AccountsContextT } from "@/accounts/types/context"; +import { sharedCryptoWorker } from "@/base/crypto"; import type { B64EncryptionResult } from "@/base/crypto/libsodium"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { ensure } from "@/utils/ensure"; import { VerticallyCentered } from "@ente/shared/components/Container"; diff --git a/web/packages/accounts/services/passkey.ts b/web/packages/accounts/services/passkey.ts index 5da030fb54..901476a67b 100644 --- a/web/packages/accounts/services/passkey.ts +++ b/web/packages/accounts/services/passkey.ts @@ -1,6 +1,6 @@ import { clientPackageName, isDesktop } from "@/base/app"; +import { sharedCryptoWorker } from "@/base/crypto"; import { encryptToB64, generateEncryptionKey } from "@/base/crypto/libsodium"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; import { clientPackageHeader, HTTPError } from "@/base/http"; import log from "@/base/log"; import { accountsAppOrigin, apiURL } from "@/base/origins"; diff --git a/web/packages/accounts/services/srp.ts b/web/packages/accounts/services/srp.ts index 044a045ce9..c593556fc1 100644 --- a/web/packages/accounts/services/srp.ts +++ b/web/packages/accounts/services/srp.ts @@ -1,5 +1,5 @@ import type { UserVerificationResponse } from "@/accounts/types/user"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import log from "@/base/log"; import { generateLoginSubKey } from "@ente/shared/crypto/helpers"; import InMemoryStore, { MS_KEYS } from "@ente/shared/storage/InMemoryStore"; diff --git a/web/packages/accounts/utils/srp.ts b/web/packages/accounts/utils/srp.ts index 21d1fa4ad3..3f9a9d6efc 100644 --- a/web/packages/accounts/utils/srp.ts +++ b/web/packages/accounts/utils/srp.ts @@ -1,4 +1,4 @@ -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import { generateLoginSubKey } from "@ente/shared/crypto/helpers"; import type { KeyAttributes } from "@ente/shared/user/types"; import { generateSRPSetupAttributes } from "../services/srp"; diff --git a/web/packages/base/crypto/ente.ts b/web/packages/base/crypto/ente.ts index 7a761ab0c5..eaa8482eab 100644 --- a/web/packages/base/crypto/ente.ts +++ b/web/packages/base/crypto/ente.ts @@ -47,6 +47,7 @@ * recommendation though (in circumstances where we create more crypto workers * instead of using the shared one). */ +import { sharedCryptoWorker } from "."; import { assertionFailed } from "../assert"; import { inWorker } from "../env"; import * as ei from "./ente-impl"; @@ -56,7 +57,6 @@ import type { EncryptBytes, EncryptJSON, } from "./types"; -import { sharedCryptoWorker } from "./worker"; /** * Some of these functions have not yet been needed on the main thread, and for diff --git a/web/packages/base/crypto/worker/index.ts b/web/packages/base/crypto/index.ts similarity index 87% rename from web/packages/base/crypto/worker/index.ts rename to web/packages/base/crypto/index.ts index 2443080aaf..511cb3f9ab 100644 --- a/web/packages/base/crypto/worker/index.ts +++ b/web/packages/base/crypto/index.ts @@ -1,11 +1,6 @@ import { ComlinkWorker } from "@/base/worker/comlink-worker"; import type { CryptoWorker } from "./worker"; -/** - * Reexport the type for easier import in call sites. - */ -export { CryptoWorker } from "./worker"; - /** * Cached instance of the {@link ComlinkWorker} that wraps our web worker. */ diff --git a/web/packages/base/crypto/worker/worker.ts b/web/packages/base/crypto/worker.ts similarity index 98% rename from web/packages/base/crypto/worker/worker.ts rename to web/packages/base/crypto/worker.ts index df9b507f8c..32b62b327a 100644 --- a/web/packages/base/crypto/worker/worker.ts +++ b/web/packages/base/crypto/worker.ts @@ -1,7 +1,7 @@ import { expose } from "comlink"; import type { StateAddress } from "libsodium-wrappers"; -import * as ei from "../ente-impl"; -import * as libsodium from "../libsodium"; +import * as ei from "./ente-impl"; +import * as libsodium from "./libsodium"; /** * A web worker that exposes some of the functions defined in either the Ente diff --git a/web/packages/new/photos/services/download.ts b/web/packages/new/photos/services/download.ts index 9b54a10bae..a5b06c27cf 100644 --- a/web/packages/new/photos/services/download.ts +++ b/web/packages/new/photos/services/download.ts @@ -3,7 +3,8 @@ import { isDesktop } from "@/base/app"; import { blobCache, type BlobCache } from "@/base/blob-cache"; -import { type CryptoWorker, sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; +import { type CryptoWorker } from "@/base/crypto/worker"; import log from "@/base/log"; import { customAPIOrigin } from "@/base/origins"; import { FileType } from "@/media/file-type"; diff --git a/web/packages/shared/components/VerifyMasterPasswordForm.tsx b/web/packages/shared/components/VerifyMasterPasswordForm.tsx index c2f962c989..86eff40417 100644 --- a/web/packages/shared/components/VerifyMasterPasswordForm.tsx +++ b/web/packages/shared/components/VerifyMasterPasswordForm.tsx @@ -1,5 +1,5 @@ import type { SRPAttributes } from "@/accounts/types/srp"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import log from "@/base/log"; import { Input, type ButtonProps } from "@mui/material"; import { t } from "i18next"; diff --git a/web/packages/shared/crypto/helpers.ts b/web/packages/shared/crypto/helpers.ts index 6b0acb7a35..c7165713ff 100644 --- a/web/packages/shared/crypto/helpers.ts +++ b/web/packages/shared/crypto/helpers.ts @@ -1,5 +1,5 @@ import { setRecoveryKey } from "@/accounts/api/user"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; +import { sharedCryptoWorker } from "@/base/crypto"; import log from "@/base/log"; import { LS_KEYS, diff --git a/web/packages/shared/user/index.ts b/web/packages/shared/user/index.ts index 69a9780b47..d23370f2c0 100644 --- a/web/packages/shared/user/index.ts +++ b/web/packages/shared/user/index.ts @@ -1,5 +1,5 @@ +import { sharedCryptoWorker } from "@/base/crypto"; import type { B64EncryptionResult } from "@/base/crypto/libsodium"; -import { sharedCryptoWorker } from "@/base/crypto/worker"; import { CustomError } from "@ente/shared/error"; import { getKey, SESSION_KEYS } from "@ente/shared/storage/sessionStorage"; From 215899f35a1754ae4fc60162dba0301fd3e7b01a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Sat, 10 Aug 2024 20:28:35 +0530 Subject: [PATCH 168/211] Nicer --- web/packages/base/crypto/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/base/crypto/index.ts b/web/packages/base/crypto/index.ts index 511cb3f9ab..736978f7ff 100644 --- a/web/packages/base/crypto/index.ts +++ b/web/packages/base/crypto/index.ts @@ -18,6 +18,6 @@ export const sharedCryptoWorker = async () => */ export const createComlinkCryptoWorker = () => new ComlinkWorker( - "Crypto", + "crypto", new Worker(new URL("worker.ts", import.meta.url)), ); From 62a979656f0a2e2a97351a97c5b1551aa9422016 Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Mon, 12 Aug 2024 00:32:10 +0000 Subject: [PATCH 169/211] New Crowdin translations by GitHub Action --- .../base/locales/el-GR/translation.json | 30 +++++++++---------- .../base/locales/es-ES/translation.json | 14 ++++----- .../base/locales/fr-FR/translation.json | 6 ++-- .../base/locales/nl-NL/translation.json | 22 +++++++------- .../base/locales/pl-PL/translation.json | 2 +- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/web/packages/base/locales/el-GR/translation.json b/web/packages/base/locales/el-GR/translation.json index a8a9b6cd00..4f00dd9620 100644 --- a/web/packages/base/locales/el-GR/translation.json +++ b/web/packages/base/locales/el-GR/translation.json @@ -31,7 +31,7 @@ "VERIFY_PASSPHRASE": "Σύνδεση", "INCORRECT_PASSPHRASE": "Λάθος κωδικός πρόσβασης", "ENTER_ENC_PASSPHRASE": "Εισάγετε έναν κωδικό πρόσβασης που μπορούμε να χρησιμοποιήσουμε για την κρυπτογράφηση των δεδομένων σας", - "PASSPHRASE_DISCLAIMER": "", + "PASSPHRASE_DISCLAIMER": "Δεν αποθηκεύουμε τον κωδικό πρόσβασής σας, επομένως, αν τον ξεχάσετε, δεν θα μπορέσουμε να σας βοηθήσουμε να ανακτήσετε τα δεδομένα σας χωρίς κλειδί ανάκτησης.", "WELCOME_TO_ENTE_HEADING": "Καλώς ήρθατε στο ", "WELCOME_TO_ENTE_SUBHEADING": "", "WHERE_YOUR_BEST_PHOTOS_LIVE": "Εκεί όπου υπάρχουν οι καλύτερες φωτογραφίες σας", @@ -47,7 +47,7 @@ "ENTER_FILE_NAME": "Όνομα αρχείου", "CLOSE": "Κλείσιμο", "NO": "Όχι", - "NOTHING_HERE": "", + "NOTHING_HERE": "Τίποτα για να δείτε εδώ ακόμα 👀", "upload": "Μεταφόρτωση", "import": "Εισαγωγή", "ADD_PHOTOS": "Προσθήκη φωτογραφιών", @@ -58,14 +58,14 @@ "FILE_UPLOAD": "Μεταφόρτωση Αρχείου", "UPLOAD_STAGE_MESSAGE": { "0": "Προετοιμασία για μεταφόρτωση", - "1": "", - "2": "", - "3": "", + "1": "Ανάγνωση αρχείων μεταδεδομένων google", + "2": "{{uploadCounter.finished, number}} / {{uploadCounter.total, number}} μεταδεδομένα των αρχείων εξήχθησαν", + "3": "{{uploadCounter.finished, number}} / {{uploadCounter.total, number}} αρχεία επεξεργάζονται", "4": "Ακύρωση των υπολειπόμενων μεταφορτώσεων", "5": "Το αντίγραφο ασφαλείας ολοκληρώθηκε" }, "FILE_NOT_UPLOADED_LIST": "Τα ακόλουθα αρχεία δεν μεταφορτώθηκαν", - "INITIAL_LOAD_DELAY_WARNING": "", + "INITIAL_LOAD_DELAY_WARNING": "Η πρώτη φόρτωση μπορεί να πάρει κάποιο χρόνο", "USER_DOES_NOT_EXIST": "Συγγνώμη, δε βρέθηκε χρήστης με αυτή τη διεύθυνση ηλ. ταχυδρομείου", "NO_ACCOUNT": "Δεν έχετε λογαριασμό", "ACCOUNT_EXISTS": "Έχετε ήδη λογαριασμό", @@ -73,19 +73,19 @@ "DOWNLOAD": "Λήψη", "DOWNLOAD_OPTION": "Λήψη (D)", "DOWNLOAD_FAVORITES": "Λήψη αγαπημένων", - "DOWNLOAD_UNCATEGORIZED": "", + "DOWNLOAD_UNCATEGORIZED": "Λήψη των μη κατηγοριοποιημένων", "DOWNLOAD_HIDDEN_ITEMS": "Λήψη κρυφών αντικειμένων", "COPY_OPTION": "Αντιγραφή ως PNG (Ctrl/Cmd - C)", "TOGGLE_FULLSCREEN": "Εναλλαγή πλήρους οθόνης (F)", - "ZOOM_IN_OUT": "", + "ZOOM_IN_OUT": "Μεγέθυνση μέσα/έξω", "PREVIOUS": "Προηγούμενο (←)", "NEXT": "Επόμενο (→)", "title_photos": "Ente Photos", "title_auth": "Ente Auth", - "title_accounts": "", + "title_accounts": "Λογαριασμοί Ente", "UPLOAD_FIRST_PHOTO": "Μεταφορτώστε την πρώτη σας φωτογραφία", "IMPORT_YOUR_FOLDERS": "Εισάγετε τους φακέλους σας", - "UPLOAD_DROPZONE_MESSAGE": "", + "UPLOAD_DROPZONE_MESSAGE": "Σύρετε για να δημιουργήσετε αντίγραφα ασφαλείας των αρχείων σας", "WATCH_FOLDER_DROPZONE_MESSAGE": "", "TRASH_FILES_TITLE": "Διαγραφή αρχείων;", "TRASH_FILE_TITLE": "Διαγραφή αρχείου;", @@ -102,24 +102,24 @@ "UPLOAD_STRATEGY_COLLECTION_PER_FOLDER": "Ξεχωριστά άλμπουμ", "SESSION_EXPIRED_MESSAGE": "Η συνεδρία σας έληξε, παρακαλούμε συνδεθείτε ξανά για να συνεχίσετε", "SESSION_EXPIRED": "Η συνεδρία έληξε", - "PASSWORD_GENERATION_FAILED": "", + "PASSWORD_GENERATION_FAILED": "Ο περιηγητής σας δεν μπόρεσε να δημιουργήσει ένα ισχυρό κλειδί που να πληροί τα πρότυπα κρυπτογράφησης του Ente, παρακαλώ δοκιμάστε χρησιμοποιώντας την εφαρμογή ή άλλον περιηγητή", "CHANGE_PASSWORD": "Αλλαγή κωδικού πρόσβασής", "password_changed_elsewhere": "", - "password_changed_elsewhere_message": "", + "password_changed_elsewhere_message": "Παρακαλώ συνδεθείτε ξανά σε αυτήν τη συσκευή για να χρησιμοποιήσετε το νέο σας κωδικό πρόσβασης για αυθεντικοποίηση.", "GO_BACK": "Επιστροφή", "RECOVERY_KEY": "Κλειδί ανάκτησης", "SAVE_LATER": "Κάντε το αργότερα", "SAVE": "Αποθήκευση Κλειδιού", "RECOVERY_KEY_DESCRIPTION": "Εάν ξεχάσετε τον κωδικό πρόσβασής σας, ο μόνος τρόπος για να ανακτήσετε τα δεδομένα σας είναι με αυτό το κλειδί.", "RECOVER_KEY_GENERATION_FAILED": "Δεν ήταν δυνατή η δημιουργία κωδικού ανάκτησης, παρακαλώ προσπαθήστε ξανά", - "KEY_NOT_STORED_DISCLAIMER": "", + "KEY_NOT_STORED_DISCLAIMER": "Δεν αποθηκεύουμε αυτό το κλειδί, οπότε παρακαλώ αποθηκεύστε αυτό το κλειδί σε μια ασφαλή τοποθεσία", "FORGOT_PASSWORD": "Ξέχασα τον κωδικό πρόσβασης", "RECOVER_ACCOUNT": "Ανάκτηση λογαριασμού", "RECOVERY_KEY_HINT": "Κλειδί ανάκτησης", "RECOVER": "Ανάκτηση", - "NO_RECOVERY_KEY": "", + "NO_RECOVERY_KEY": "Χωρίς κλειδί ανάκτησης;", "INCORRECT_RECOVERY_KEY": "Εσφαλμένο κλειδί ανάκτησης", - "SORRY": "", + "SORRY": "Συγνώμη", "NO_RECOVERY_KEY_MESSAGE": "", "NO_TWO_FACTOR_RECOVERY_KEY_MESSAGE": "Παρακαλώ αφήστε ένα μήνυμα ηλ. ταχυδρομείου στο {{emailID}} από την καταχωρημένη διεύθυνση σας", "CONTACT_SUPPORT": "Επικοινωνήστε με την υποστήριξη", diff --git a/web/packages/base/locales/es-ES/translation.json b/web/packages/base/locales/es-ES/translation.json index 3831ea2a4b..4d6090e7f1 100644 --- a/web/packages/base/locales/es-ES/translation.json +++ b/web/packages/base/locales/es-ES/translation.json @@ -220,7 +220,7 @@ "FILE_NAME": "Nombre del archivo", "THING": "Contenido", "FILE_CAPTION": "Descripción", - "FILE_TYPE": "", + "FILE_TYPE": "Tipo de archivo", "CLIP": "Magia" }, "photos_count_zero": "No hay recuerdos", @@ -250,7 +250,7 @@ "DISABLE_MAPS": "¿Desactivar Mapas?", "ENABLE_MAP_DESCRIPTION": "", "DISABLE_MAP_DESCRIPTION": "", - "DISABLE_MAP": "", + "DISABLE_MAP": "Desactivar mapa", "DETAILS": "Detalles", "view_exif": "Ver todos los datos de Exif", "no_exif": "No hay datos Exif", @@ -280,7 +280,7 @@ "EXPORT_DATA": "Exportar datos", "select_folder": "Seleccionar carpeta", "select_zips": "", - "faq": "", + "faq": "Preguntas Frecuentes", "takeout_hint": "", "DESTINATION": "Destinación", "START": "Inicio", @@ -385,7 +385,7 @@ "OWNER": "Propietario", "COLLABORATORS": "Colaboradores", "ADD_MORE": "Añadir más", - "VIEWERS": "", + "VIEWERS": "Espectadores", "OR_ADD_EXISTING": "O elige uno existente", "REMOVE_PARTICIPANT_MESSAGE": "", "NOT_FOUND": "404 - No Encontrado", @@ -547,9 +547,9 @@ "HIDE": "Ocultar", "UNHIDE": "Mostrar", "UNHIDE_TO_COLLECTION": "Hacer visible al álbum", - "SORT_BY": "", - "NEWEST_FIRST": "", - "OLDEST_FIRST": "", + "SORT_BY": "Ordenar por", + "NEWEST_FIRST": "Más recientes primero", + "OLDEST_FIRST": "Más antiguos primero", "CONVERSION_FAILED_NOTIFICATION_MESSAGE": "", "SELECT_COLLECTION": "", "PIN_ALBUM": "", diff --git a/web/packages/base/locales/fr-FR/translation.json b/web/packages/base/locales/fr-FR/translation.json index 241424f558..9600ac8ea4 100644 --- a/web/packages/base/locales/fr-FR/translation.json +++ b/web/packages/base/locales/fr-FR/translation.json @@ -153,10 +153,10 @@ "CURRENT_USAGE": "L'utilisation actuelle est de {{usage}}", "TWO_MONTHS_FREE": "Obtenir 2 mois gratuits sur les plans annuels", "POPULAR": "Populaire", - "free_plan_option": "", - "free_plan_description": "", + "free_plan_option": "Continuer avec le plan gratuit", + "free_plan_description": "{{storage}} gratuit pour toujours", "active": "Actif", - "subscription_info_free": "", + "subscription_info_free": "Vous êtes sur un plan gratuit", "subscription_info_family": "Vous êtes sur le plan famille géré par", "subscription_info_expired": "Votre abonnement a expiré, veuillez le renouveler ", "subscription_info_renewal_cancelled": "Votre abonnement sera annulé le {{date, date}}", diff --git a/web/packages/base/locales/nl-NL/translation.json b/web/packages/base/locales/nl-NL/translation.json index f96710c822..89abea4894 100644 --- a/web/packages/base/locales/nl-NL/translation.json +++ b/web/packages/base/locales/nl-NL/translation.json @@ -153,10 +153,10 @@ "CURRENT_USAGE": "Huidig gebruik is {{usage}}", "TWO_MONTHS_FREE": "Krijg 2 maanden gratis op jaarlijkse abonnementen", "POPULAR": "Populair", - "free_plan_option": "", - "free_plan_description": "", + "free_plan_option": "Doorgaan met gratis account", + "free_plan_description": "{{storage}} voor altijd gratis", "active": "Actief", - "subscription_info_free": "", + "subscription_info_free": "Je hebt een gratis abonnement", "subscription_info_family": "U hebt een familieplan dat beheerd wordt door", "subscription_info_expired": "Uw abonnement is verlopen, gelieve vernieuwen", "subscription_info_renewal_cancelled": "Uw abonnement loopt af op {{date, date}}", @@ -231,9 +231,9 @@ "SELECTED": "geselecteerd", "PEOPLE": "Personen", "indexing_scheduled": "Indexering is gepland...", - "indexing_photos": "", - "indexing_people": "", - "indexing_done": "", + "indexing_photos": "Analyseren van foto's ({{nSyncedFiles, number}} / {{nTotalFiles, number}})", + "indexing_people": "Mensen analyseren in {{nSyncedFiles, number}} photos...", + "indexing_done": "{{nSyncedFiles, number}} foto's geanalyseerd", "UNIDENTIFIED_FACES": "Ongeïdentificeerde gezichten", "OBJECTS": "objecten", "TEXT": "tekst", @@ -279,9 +279,9 @@ "TWO_FACTOR_DISABLE_FAILED": "Uitschakelen van tweestapsverificatie is mislukt, probeer het opnieuw", "EXPORT_DATA": "Gegevens exporteren", "select_folder": "Map selecteren", - "select_zips": "", - "faq": "", - "takeout_hint": "", + "select_zips": "Selecteer zip", + "faq": "Veelgestelde vragen", + "takeout_hint": "Pak alle zips uit in dezelfde map en upload die. Of upload de zips direct als aparte mappen. Zie de FAQ voor meer informatie.", "DESTINATION": "Bestemming", "START": "Start", "LAST_EXPORT_TIME": "Tijd laatste export", @@ -294,7 +294,7 @@ "LIVE_PHOTOS_DETECTED": "De foto en video bestanden van je Live Photos zijn samengevoegd tot één enkel bestand", "RETRY_FAILED": "Probeer mislukte uploads nogmaals", "FAILED_UPLOADS": "Mislukte uploads ", - "failed_uploads_hint": "", + "failed_uploads_hint": "Er zal een optie zijn om deze opnieuw te proberen wanneer het uploaden voltooid is", "SKIPPED_FILES": "Genegeerde uploads", "THUMBNAIL_GENERATION_FAILED_UPLOADS": "Thumbnail generatie mislukt", "UNSUPPORTED_FILES": "Niet-ondersteunde bestanden", @@ -412,7 +412,7 @@ "DOWNLOAD_UPLOAD_LOGS": "Logboeken voor foutmeldingen", "file": "Bestand", "folder": "Map", - "google_takeout": "", + "google_takeout": "Google takeout", "DEDUPLICATE_FILES": "Dubbele bestanden verwijderen", "NO_DUPLICATES_FOUND": "Je hebt geen dubbele bestanden die kunnen worden gewist", "FILES": "bestanden", diff --git a/web/packages/base/locales/pl-PL/translation.json b/web/packages/base/locales/pl-PL/translation.json index 91056c5443..26c808e132 100644 --- a/web/packages/base/locales/pl-PL/translation.json +++ b/web/packages/base/locales/pl-PL/translation.json @@ -412,7 +412,7 @@ "DOWNLOAD_UPLOAD_LOGS": "Logi debugowania", "file": "Plik", "folder": "Folder", - "google_takeout": "Paczka danych Google", + "google_takeout": "Google Takeout", "DEDUPLICATE_FILES": "Odduplikuj pliki", "NO_DUPLICATES_FOUND": "Nie masz zduplikowanych plików, które można wyczyścić", "FILES": "pliki", From 66726846dafc5d89b0fd54582892b010b0ec6624 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 10:22:17 +0530 Subject: [PATCH 170/211] d1 --- docs/docs/.vitepress/sidebar.ts | 4 ++ docs/docs/photos/features/advanced-search.md | 40 ++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 docs/docs/photos/features/advanced-search.md diff --git a/docs/docs/.vitepress/sidebar.ts b/docs/docs/.vitepress/sidebar.ts index 11a2c30119..9b2a23bc46 100644 --- a/docs/docs/.vitepress/sidebar.ts +++ b/docs/docs/.vitepress/sidebar.ts @@ -10,6 +10,10 @@ export const sidebar = [ text: "Features", collapsed: true, items: [ + { + text: "Advanced search", + link: "/photos/features/advanced-search", + }, { text: "Albums", link: "/photos/features/albums" }, { text: "Archiving", link: "/photos/features/archive" }, { diff --git a/docs/docs/photos/features/advanced-search.md b/docs/docs/photos/features/advanced-search.md new file mode 100644 index 0000000000..939f37a177 --- /dev/null +++ b/docs/docs/photos/features/advanced-search.md @@ -0,0 +1,40 @@ +> [!NOTE] +> +> This document describes a beta feature that will be present in an upcoming +> release. + +Ente supports on device machine learning. This allows you to use the latest +advances in AI in a privacy preserving manner. + +* You can search for your photos by the **faces** of the people in them. Ente + will show you all the faces in a photo, and will also try to group similar + faces together to create clusters of people so that you can give them names, + and quickly find all photos with a given person in them. + +* You can search for your photos by typing natural language descriptions of + them. For example, you can search for "night", "by the seaside", "the red + motorcycle next to a fountain", etc. Within the app, this ability is + sometimes referred to as **magic search**. + +* We will build on this foundation to add more forms of advanced search. + +You can enable face and magic search in the app's preferences on either the +mobile app or the desktop app. + +If you have a big library, we recommend enabling this on the desktop app first, +because it can index your existing photos faster (The app needs to download your +originals to index them which can happen faster over WiFi, and indexing is also +faster on your computer as compared to your mobile device). + +Once your existing photos have been indexed, then you can use either. The mobile +app is fast enough to easily and seamlessly index the new photos that you take. + +> [!TIP] +> +> Even for the initial indexing, you don't necessarily need the desktop app, it +> just will be a bit faster. + +The indexes are synced across all your devices automatically using the same end-to-end encypted security that we use for syncing your photos. + +Note that the desktop app does not currently support modifying the automatically +generated face groupings, that is only supported by the mobile app. From a70a3c7078fcdacbd458f356a332b9fe3df93c34 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 10:27:11 +0530 Subject: [PATCH 171/211] edit --- docs/docs/photos/features/advanced-search.md | 29 +++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/docs/docs/photos/features/advanced-search.md b/docs/docs/photos/features/advanced-search.md index 939f37a177..da8b8ad7be 100644 --- a/docs/docs/photos/features/advanced-search.md +++ b/docs/docs/photos/features/advanced-search.md @@ -1,22 +1,30 @@ +--- +title: Advanced search +description: + On device machine learning in Ente for face and natural language search +--- + +# Advanced search + > [!NOTE] > > This document describes a beta feature that will be present in an upcoming > release. -Ente supports on device machine learning. This allows you to use the latest +Ente supports on-device machine learning. This allows you to use the latest advances in AI in a privacy preserving manner. -* You can search for your photos by the **faces** of the people in them. Ente +- You can search for your photos by the **faces** of the people in them. Ente will show you all the faces in a photo, and will also try to group similar faces together to create clusters of people so that you can give them names, and quickly find all photos with a given person in them. -* You can search for your photos by typing natural language descriptions of - them. For example, you can search for "night", "by the seaside", "the red - motorcycle next to a fountain", etc. Within the app, this ability is - sometimes referred to as **magic search**. +- You can search for your photos by typing natural language descriptions of + them. For example, you can search for "night", "by the seaside", or "the red + motorcycle next to a fountain". Within the app, this ability is sometimes + referred to as **magic search**. -* We will build on this foundation to add more forms of advanced search. +- We will build on this foundation to add more forms of advanced search. You can enable face and magic search in the app's preferences on either the mobile app or the desktop app. @@ -31,10 +39,11 @@ app is fast enough to easily and seamlessly index the new photos that you take. > [!TIP] > -> Even for the initial indexing, you don't necessarily need the desktop app, it -> just will be a bit faster. +> Even for the initial indexing, you don't necessarily need the desktop app, it +> just will be a bit faster. -The indexes are synced across all your devices automatically using the same end-to-end encypted security that we use for syncing your photos. +The indexes are synced across all your devices automatically using the same +end-to-end encypted security that we use for syncing your photos. Note that the desktop app does not currently support modifying the automatically generated face groupings, that is only supported by the mobile app. From 83e9b18ebad0c010eec2a90eebf34a9c70d19f0e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 10:38:05 +0530 Subject: [PATCH 172/211] Update link --- web/packages/new/photos/components/MLSettings.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/web/packages/new/photos/components/MLSettings.tsx b/web/packages/new/photos/components/MLSettings.tsx index 2e5a0c6def..27512e91d8 100644 --- a/web/packages/new/photos/components/MLSettings.tsx +++ b/web/packages/new/photos/components/MLSettings.tsx @@ -155,14 +155,14 @@ interface EnableMLProps { } const EnableML: React.FC = ({ onEnable }) => { - // TODO-ML: Update link. - const moreDetails = () => openURL("https://ente.io/blog/desktop-ml-beta"); + const moreDetails = () => + openURL("https://help.ente.io/photos/features/advanced-search"); return ( {pt( - "Enable ML (Machine Learning) for face recognition, magic search and other advanced search features", + "Ente supports on-device machine learning for face recognition, magic search and other advanced search features", )} @@ -176,7 +176,7 @@ const EnableML: React.FC = ({ onEnable }) => { {pt( - 'Magic search allows to search photos by their contents (e.g. "car", "red car" or even "ferrari")', + 'Magic search allows to search photos by their contents, e.g. "car", "red car" or even very specific things like "Ferrari".', )} From 92adb3ad6f97cecbefa3302d2969ce0c5c343d3a Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 11:05:50 +0530 Subject: [PATCH 173/211] Remove old one --- .../components/Sidebar/AdvancedSettings.tsx | 45 +++----------- .../src/components/Sidebar/MapSetting.tsx | 11 +++- .../src/components/Sidebar/Preferences.tsx | 25 +++++--- .../photos/src/components/Sidebar/index.tsx | 4 +- .../photos/src/components/Sidebar/types.ts | 8 +++ .../new/photos/components/MLSettingsBeta.tsx | 60 ------------------- 6 files changed, 44 insertions(+), 109 deletions(-) create mode 100644 web/apps/photos/src/components/Sidebar/types.ts delete mode 100644 web/packages/new/photos/components/MLSettingsBeta.tsx diff --git a/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx b/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx index 8980c8ed5f..7240b47d17 100644 --- a/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx +++ b/web/apps/photos/src/components/Sidebar/AdvancedSettings.tsx @@ -1,27 +1,20 @@ -import { isDesktop } from "@/base/app"; import { EnteDrawer } from "@/base/components/EnteDrawer"; import { MenuItemGroup, MenuSectionTitle } from "@/base/components/Menu"; import { Titlebar } from "@/base/components/Titlebar"; -import { pt } from "@/base/i18n"; -import { MLSettingsBeta } from "@/new/photos/components/MLSettingsBeta"; -import { canEnableML } from "@/new/photos/services/ml"; import { EnteMenuItem } from "@ente/shared/components/Menu/EnteMenuItem"; -import ChevronRight from "@mui/icons-material/ChevronRight"; -import ScienceIcon from "@mui/icons-material/Science"; import { Box, DialogProps, Stack } from "@mui/material"; import { t } from "i18next"; import { AppContext } from "pages/_app"; -import { useContext, useEffect, useState } from "react"; +import React, { useContext } from "react"; +import type { SettingsDrawerProps } from "./types"; -export default function AdvancedSettings({ open, onClose, onRootClose }) { +export const AdvancedSettings: React.FC = ({ + open, + onClose, + onRootClose, +}) => { const appContext = useContext(AppContext); - const [showMLSettings, setShowMLSettings] = useState(false); - const [openMLSettings, setOpenMLSettings] = useState(false); - - useEffect(() => { - if (isDesktop) void canEnableML().then(setShowMLSettings); - }, []); const handleRootClose = () => { onClose(); onRootClose(); @@ -71,30 +64,8 @@ export default function AdvancedSettings({ open, onClose, onRootClose }) { /> - - {showMLSettings && ( - - } - /> - - } - onClick={() => setOpenMLSettings(true)} - label={pt("Face and magic search")} - /> - - - )} - - setOpenMLSettings(false)} - onRootClose={handleRootClose} - /> ); -} +}; diff --git a/web/apps/photos/src/components/Sidebar/MapSetting.tsx b/web/apps/photos/src/components/Sidebar/MapSetting.tsx index 29acee238c..a64d64c9eb 100644 --- a/web/apps/photos/src/components/Sidebar/MapSetting.tsx +++ b/web/apps/photos/src/components/Sidebar/MapSetting.tsx @@ -13,11 +13,16 @@ import { } from "@mui/material"; import { t } from "i18next"; import { AppContext } from "pages/_app"; -import { useContext, useEffect, useState } from "react"; +import React, { useContext, useEffect, useState } from "react"; import { Trans } from "react-i18next"; import { getMapEnabledStatus } from "services/userService"; +import type { SettingsDrawerProps } from "./types"; -export default function MapSettings({ open, onClose, onRootClose }) { +export const MapSettings: React.FC = ({ + open, + onClose, + onRootClose, +}) => { const { mapEnabled, updateMapEnabled } = useContext(AppContext); const [modifyMapEnabledView, setModifyMapEnabledView] = useState(false); @@ -87,7 +92,7 @@ export default function MapSettings({ open, onClose, onRootClose }) { /> ); -} +}; const ModifyMapEnabled = ({ open, onClose, onRootClose, mapEnabled }) => { const { somethingWentWrong, updateMapEnabled } = useContext(AppContext); diff --git a/web/apps/photos/src/components/Sidebar/Preferences.tsx b/web/apps/photos/src/components/Sidebar/Preferences.tsx index 000a1e44cc..05d72c92ae 100644 --- a/web/apps/photos/src/components/Sidebar/Preferences.tsx +++ b/web/apps/photos/src/components/Sidebar/Preferences.tsx @@ -1,3 +1,4 @@ +import { isDesktop } from "@/base/app"; import { EnteDrawer } from "@/base/components/EnteDrawer"; import { MenuItemGroup, MenuSectionTitle } from "@/base/components/Menu"; import { Titlebar } from "@/base/components/Titlebar"; @@ -9,7 +10,7 @@ import { type SupportedLocale, } from "@/base/i18n"; import { MLSettings } from "@/new/photos/components/MLSettings"; -import { isMLSupported } from "@/new/photos/services/ml"; +import { canEnableML } from "@/new/photos/services/ml"; import { EnteMenuItem } from "@ente/shared/components/Menu/EnteMenuItem"; import ChevronRight from "@mui/icons-material/ChevronRight"; import ScienceIcon from "@mui/icons-material/Science"; @@ -17,17 +18,27 @@ import { Box, DialogProps, Stack } from "@mui/material"; import DropdownInput from "components/DropdownInput"; import { t } from "i18next"; import { AppContext } from "pages/_app"; -import { useContext, useState } from "react"; -import AdvancedSettings from "./AdvancedSettings"; -import MapSettings from "./MapSetting"; +import React, { useContext, useEffect, useState } from "react"; +import { AdvancedSettings } from "./AdvancedSettings"; +import { MapSettings } from "./MapSetting"; +import type { SettingsDrawerProps } from "./types"; -export default function Preferences({ open, onClose, onRootClose }) { +export const Preferences: React.FC = ({ + open, + onClose, + onRootClose, +}) => { const appContext = useContext(AppContext); const [advancedSettingsView, setAdvancedSettingsView] = useState(false); const [mapSettingsView, setMapSettingsView] = useState(false); + const [showMLSettings, setShowMLSettings] = useState(false); const [openMLSettings, setOpenMLSettings] = useState(false); + useEffect(() => { + if (isDesktop) void canEnableML().then(setShowMLSettings); + }, []); + const openAdvancedSettings = () => setAdvancedSettingsView(true); const closeAdvancedSettings = () => setAdvancedSettingsView(false); @@ -75,7 +86,7 @@ export default function Preferences({ open, onClose, onRootClose }) { endIcon={} label={t("ADVANCED")} /> - {isMLSupported && ( + {showMLSettings && ( ); -} +}; const LanguageSelector = () => { const locale = getLocaleInUse(); diff --git a/web/apps/photos/src/components/Sidebar/index.tsx b/web/apps/photos/src/components/Sidebar/index.tsx index ccf921c854..9c3a19d8c7 100644 --- a/web/apps/photos/src/components/Sidebar/index.tsx +++ b/web/apps/photos/src/components/Sidebar/index.tsx @@ -50,7 +50,7 @@ import isElectron from "is-electron"; import { useRouter } from "next/router"; import { AppContext } from "pages/_app"; import { GalleryContext } from "pages/gallery"; -import { +import React, { MouseEventHandler, useContext, useEffect, @@ -78,7 +78,7 @@ import { getDownloadAppMessage } from "utils/ui"; import { isFamilyAdmin, isPartOfFamily } from "utils/user/family"; import { testUpload } from "../../../tests/upload.test"; import { MemberSubscriptionManage } from "../MemberSubscriptionManage"; -import Preferences from "./Preferences"; +import { Preferences } from "./Preferences"; import SubscriptionCard from "./SubscriptionCard"; interface Iprops { diff --git a/web/apps/photos/src/components/Sidebar/types.ts b/web/apps/photos/src/components/Sidebar/types.ts new file mode 100644 index 0000000000..e9eebadece --- /dev/null +++ b/web/apps/photos/src/components/Sidebar/types.ts @@ -0,0 +1,8 @@ +export interface SettingsDrawerProps { + /** If `true`, then this drawer page is shown. */ + open: boolean; + /** Called when the user wants to go back from this drawer page. */ + onClose: () => void; + /** Called when the user wants to close the entire stack of drawers. */ + onRootClose: () => void; +} diff --git a/web/packages/new/photos/components/MLSettingsBeta.tsx b/web/packages/new/photos/components/MLSettingsBeta.tsx deleted file mode 100644 index 2f9bae19f4..0000000000 --- a/web/packages/new/photos/components/MLSettingsBeta.tsx +++ /dev/null @@ -1,60 +0,0 @@ -import { EnteDrawer } from "@/base/components/EnteDrawer"; -import { Titlebar } from "@/base/components/Titlebar"; -import { pt, ut } from "@/base/i18n"; -import { Box, Stack, Typography, type DialogProps } from "@mui/material"; -import React from "react"; - -interface MLSettingsBetaProps { - /** If `true`, then this drawer page is shown. */ - open: boolean; - /** Called when the user wants to go back from this drawer page. */ - onClose: () => void; - /** Called when the user wants to close the entire stack of drawers. */ - onRootClose: () => void; -} - -export const MLSettingsBeta: React.FC = ({ - open, - onClose, - onRootClose, -}) => { - const handleRootClose = () => { - onClose(); - onRootClose(); - }; - - const handleDrawerClose: DialogProps["onClose"] = (_, reason) => { - if (reason == "backdropClick") handleRootClose(); - else onClose(); - }; - - return ( - - - - - - - - {ut( - "We're putting finishing touches, coming back soon!", - )} - - - - - - ); -}; From 2830d32c795d40ecb4c00f437ce24cf47da932ea Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 11:10:18 +0530 Subject: [PATCH 174/211] Shorten --- web/packages/new/photos/components/MLSettings.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/packages/new/photos/components/MLSettings.tsx b/web/packages/new/photos/components/MLSettings.tsx index 27512e91d8..d36067155e 100644 --- a/web/packages/new/photos/components/MLSettings.tsx +++ b/web/packages/new/photos/components/MLSettings.tsx @@ -176,7 +176,7 @@ const EnableML: React.FC = ({ onEnable }) => { {pt( - 'Magic search allows to search photos by their contents, e.g. "car", "red car" or even very specific things like "Ferrari".', + 'Magic search allows to search photos by their contents, e.g. "car", "red car", "Ferrari"', )} From 7a113a3aba8137b430f01cf6e11456579ef6e521 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 11:49:49 +0530 Subject: [PATCH 175/211] Preciser --- web/packages/new/photos/services/ml/cluster-new.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/packages/new/photos/services/ml/cluster-new.ts b/web/packages/new/photos/services/ml/cluster-new.ts index 298eafc26b..acb74639f7 100644 --- a/web/packages/new/photos/services/ml/cluster-new.ts +++ b/web/packages/new/photos/services/ml/cluster-new.ts @@ -11,7 +11,7 @@ import { dotProduct } from "./math"; */ export interface FaceCluster { /** - * A randomly generated ID to uniquely identify this cluster. + * A nanoid for this cluster. */ id: string; /** @@ -37,7 +37,7 @@ export interface FaceCluster { */ export interface Person { /** - * A randomly generated ID to uniquely identify this person. + * A nanoid for this person. */ id: string; /** From 9be305167134e0269c2b76cfb2d26a9aae56f4a1 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:13:54 +0530 Subject: [PATCH 176/211] [mobile] New translations (#2659) New translations from [Crowdin](https://crowdin.com/project/ente-photos-app) Co-authored-by: Crowdin Bot --- mobile/lib/l10n/intl_de.arb | 5 ++--- mobile/lib/l10n/intl_pl.arb | 7 +++---- mobile/lib/l10n/intl_pt.arb | 9 ++++----- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/mobile/lib/l10n/intl_de.arb b/mobile/lib/l10n/intl_de.arb index ceb5d28a44..0a16ed655e 100644 --- a/mobile/lib/l10n/intl_de.arb +++ b/mobile/lib/l10n/intl_de.arb @@ -1279,7 +1279,6 @@ "tooManyIncorrectAttempts": "Zu viele fehlerhafte Versuche", "videoInfo": "Video-Informationen", "appLockDescription": "Wähle zwischen dem Standard-Sperrbildschirm deines Gerätes und einem eigenen Sperrbildschirm mit PIN oder Passwort.", - "swipeLockEnablePreSteps": "Um die Sperre für die Wischfunktion zu aktivieren, richte bitte einen Gerätepasscode oder eine Bildschirmsperre in den Systemeinstellungen ein.", "autoLock": "Automatisches Sperren", "immediately": "Sofort", "autoLockFeatureDescription": "Zeit, nach der die App gesperrt wird, nachdem sie in den Hintergrund verschoben wurde", @@ -1291,6 +1290,6 @@ "pleaseSelectQuickLinksToRemove": "Bitte wähle die zu entfernenden schnellen Links", "removePublicLinks": "Öffentliche Links entfernen", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Hiermit werden die öffentlichen Links aller ausgewählten schnellen Links entfernt.", - "guestView": "Guest view", - "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." + "guestView": "Gastansicht", + "guestViewEnablePreSteps": "Bitte richte einen Gerätepasscode oder eine Bildschirmsperre ein, um die Gastansicht zu nutzen." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_pl.arb b/mobile/lib/l10n/intl_pl.arb index cc074a7bef..f00e9ee34d 100644 --- a/mobile/lib/l10n/intl_pl.arb +++ b/mobile/lib/l10n/intl_pl.arb @@ -644,7 +644,7 @@ "continueOnFreeTrial": "Kontynuuj bezpłatny okres próbny", "areYouSureYouWantToExit": "Czy na pewno chcesz wyjść?", "thankYou": "Dziękujemy", - "failedToVerifyPaymentStatus": "Nie udało się zweryfikować statusu płatności", + "failedToVerifyPaymentStatus": "Nie udało się zweryfikować stanu płatności", "pleaseWaitForSometimeBeforeRetrying": "Proszę poczekać chwilę przed ponowną próbą", "paymentFailedMessage": "Niestety Twoja płatność nie powiodła się. Skontaktuj się z pomocą techniczną, a my Ci pomożemy!", "youAreOnAFamilyPlan": "Jesteś w planie rodzinnym!", @@ -1279,7 +1279,6 @@ "tooManyIncorrectAttempts": "Zbyt wiele błędnych prób", "videoInfo": "Informacje Wideo", "appLockDescription": "Wybierz między domyślnym ekranem blokady urządzenia a niestandardowym ekranem blokady z kodem PIN lub hasłem.", - "swipeLockEnablePreSteps": "Aby włączyć blokadę aplikacji, należy skonfigurować hasło urządzenia lub blokadę ekranu w ustawieniach Twojego systemu.", "autoLock": "Automatyczna blokada", "immediately": "Natychmiast", "autoLockFeatureDescription": "Czas, po którym aplikacja blokuje się po umieszczeniu jej w tle", @@ -1291,6 +1290,6 @@ "pleaseSelectQuickLinksToRemove": "Prosimy wybrać szybkie linki do usunięcia", "removePublicLinks": "Usuń linki publiczne", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Spowoduje to usunięcie publicznych linków wszystkich zaznaczonych szybkich linków.", - "guestView": "Guest view", - "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." + "guestView": "Widok gościa", + "guestViewEnablePreSteps": "Aby włączyć widok gościa, należy skonfigurować hasło urządzenia lub blokadę ekranu w ustawieniach Twojego systemu." } \ No newline at end of file diff --git a/mobile/lib/l10n/intl_pt.arb b/mobile/lib/l10n/intl_pt.arb index 1259379354..b1fe399716 100644 --- a/mobile/lib/l10n/intl_pt.arb +++ b/mobile/lib/l10n/intl_pt.arb @@ -286,7 +286,7 @@ "claimFreeStorage": "Reivindicar armazenamento gratuito", "inviteYourFriends": "Convide seus amigos", "failedToFetchReferralDetails": "Não foi possível buscar os detalhes de indicação. Por favor, tente novamente mais tarde.", - "referralStep1": "Envie esse código aos seus amigos", + "referralStep1": "1. Dê este código aos seus amigos", "referralStep2": "2. Eles se inscrevem em um plano pago", "referralStep3": "3. Ambos ganham {storageInGB} GB* grátis", "referralsAreCurrentlyPaused": "Indicações estão atualmente pausadas", @@ -1118,7 +1118,7 @@ "description": "Label for the map view" }, "maps": "Mapas", - "enableMaps": "Habilitar mapa", + "enableMaps": "Habilitar Mapa", "enableMapsDesc": "Isto mostrará suas fotos em um mapa do mundo.\n\nEste mapa é hospedado pelo OpenStreetMap, e os exatos locais de suas fotos nunca são compartilhados.\n\nVocê pode desativar esse recurso a qualquer momento nas Configurações.", "quickLinks": "Links rápidos", "selectItemsToAdd": "Selecionar itens para adicionar", @@ -1279,7 +1279,6 @@ "tooManyIncorrectAttempts": "Muitas tentativas incorretas", "videoInfo": "Informação de Vídeo", "appLockDescription": "Escolha entre a tela de bloqueio padrão do seu dispositivo e uma tela de bloqueio personalizada com PIN ou senha.", - "swipeLockEnablePreSteps": "Para ativar o bloqueio por deslizar, por favor ative um método de autenticação nas configurações do sistema do seu dispositivo.", "autoLock": "Bloqueio automático", "immediately": "Imediatamente", "autoLockFeatureDescription": "Tempo após o qual o app bloqueia depois de ser colocado em segundo plano", @@ -1291,6 +1290,6 @@ "pleaseSelectQuickLinksToRemove": "Selecione links rápidos para remover", "removePublicLinks": "Remover link público", "thisWillRemovePublicLinksOfAllSelectedQuickLinks": "Isto removerá links públicos de todos os links rápidos selecionados.", - "guestView": "Guest view", - "guestViewEnablePreSteps": "To enable guest view, please setup device passcode or screen lock in your system settings." + "guestView": "Visão de convidado", + "guestViewEnablePreSteps": "Para ativar a visão de convidado, por favor ative um método de autenticação nas configurações do sistema do seu dispositivo." } \ No newline at end of file From 930c080dcfd664f6d61e99244c5544a2ef634cd0 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 12:14:45 +0530 Subject: [PATCH 177/211] [auth] New translations (#2660) New translations from [Crowdin](https://crowdin.com/project/ente-authenticator-app) Co-authored-by: Crowdin Bot --- auth/lib/l10n/arb/app_ca.arb | 21 +- auth/lib/l10n/arb/app_el.arb | 9 +- auth/lib/l10n/arb/app_fi.arb | 89 ++++++- auth/lib/l10n/arb/app_fr.arb | 22 +- auth/lib/l10n/arb/app_it.arb | 3 +- auth/lib/l10n/arb/app_nl.arb | 25 +- auth/lib/l10n/arb/app_pl.arb | 5 +- auth/lib/l10n/arb/app_pt.arb | 3 +- auth/lib/l10n/arb/app_ro.arb | 7 + auth/lib/l10n/arb/app_sk.arb | 288 ++++++++++++++++++++- auth/lib/l10n/arb/app_uk.arb | 470 +++++++++++++++++++++++++++++++++++ auth/lib/l10n/arb/app_zh.arb | 3 +- 12 files changed, 929 insertions(+), 16 deletions(-) create mode 100644 auth/lib/l10n/arb/app_uk.arb diff --git a/auth/lib/l10n/arb/app_ca.arb b/auth/lib/l10n/arb/app_ca.arb index 9e26dfeeb6..9ba17aac84 100644 --- a/auth/lib/l10n/arb/app_ca.arb +++ b/auth/lib/l10n/arb/app_ca.arb @@ -1 +1,20 @@ -{} \ No newline at end of file +{ + "account": "Compte", + "unlock": "Desbloqueja", + "recoveryKey": "Clau de recuperació", + "counterAppBarTitle": "Comptador", + "@counterAppBarTitle": { + "description": "Text shown in the AppBar of the Counter Page" + }, + "onBoardingGetStarted": "Primers passos", + "qrCode": "Codi QR", + "codeTagHint": "Etiqueta", + "accountKeyType": "Tipus de clau", + "sessionExpired": "La sessió ha caducat", + "@sessionExpired": { + "description": "Title of the dialog when the users current session is invalid/expired" + }, + "pleaseLoginAgain": "Torna a iniciar sessió", + "verifyPassword": "Verifica la contrasenya", + "pleaseWait": "Si us plau, espera..." +} \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_el.arb b/auth/lib/l10n/arb/app_el.arb index 4242d10647..0f2f8a4151 100644 --- a/auth/lib/l10n/arb/app_el.arb +++ b/auth/lib/l10n/arb/app_el.arb @@ -448,12 +448,19 @@ "rawCodeData": "Δεδομένα ακατέργαστων κωδικών", "appLock": "Κλείδωμα εφαρμογής", "noSystemLockFound": "Δεν βρέθηκε κλείδωμα συστήματος", + "toEnableAppLockPleaseSetupDevicePasscodeOrScreen": "Για να ενεργοποιήσετε το κλείδωμα εφαρμογής, παρακαλώ ορίστε τον κωδικό πρόσβασης της συσκευής ή το κλείδωμα οθόνης στις ρυθμίσεις του συστήματος σας.", "autoLock": "Αυτόματο κλείδωμα", "immediately": "Άμεσα", + "reEnterPassword": "Πληκτρολογήστε ξανά τον κωδικό πρόσβασης", + "reEnterPin": "Πληκτρολογήστε ξανά το PIN", "next": "Επόμενο", "tooManyIncorrectAttempts": "Πάρα πολλές εσφαλμένες προσπάθειες", "tapToUnlock": "Πατήστε για ξεκλείδωμα", "setNewPassword": "Ορίστε νέο κωδικό πρόσβασης", + "deviceLock": "Κλείδωμα συσκευής", "hideContent": "Απόκρυψη περιεχομένου", - "setNewPin": "Ορίστε νέο PIN" + "pinLock": "Κλείδωμα καρφιτσωμάτων", + "enterPin": "Εισαγωγή PIN", + "setNewPin": "Ορίστε νέο PIN", + "importFailureDescNew": "Αδυναμία ανάλυσης του επιλεγμένου αρχείου." } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_fi.arb b/auth/lib/l10n/arb/app_fi.arb index 2a04041475..4bf9706755 100644 --- a/auth/lib/l10n/arb/app_fi.arb +++ b/auth/lib/l10n/arb/app_fi.arb @@ -1,4 +1,7 @@ { + "account": "Käyttäjätili", + "unlock": "Avaa lukitus", + "recoveryKey": "Palautusavain", "counterAppBarTitle": "Laskuri", "@counterAppBarTitle": { "description": "Text shown in the AppBar of the Counter Page" @@ -7,11 +10,15 @@ "onBoardingGetStarted": "Aloitetaan", "setupFirstAccount": "Määritä ensimmäinen tilisi", "importScanQrCode": "Lue QR-koodi", + "qrCode": "QR-koodi", "importEnterSetupKey": "Syötä asetusavain", "importAccountPageTitle": "Syötä tilin tiedot", + "incorrectDetails": "Virheelliset tiedot", + "pleaseVerifyDetails": "Vahvista tietosi ja yritä uudelleen", "codeIssuerHint": "Myöntäjä", "codeSecretKeyHint": "Salainen avain", "codeAccountHint": "Tili (sinun@jokinosoite.com)", + "accountKeyType": "Avaimen tyyppi", "sessionExpired": "Istunto on vanheutunut", "@sessionExpired": { "description": "Title of the dialog when the users current session is invalid/expired" @@ -50,6 +57,7 @@ } }, "contactSupport": "Ota yhteyttä käyttötukeen", + "blog": "Blogi", "verifyPassword": "Vahvista salasana", "pleaseWait": "Odota hetki...", "generatingEncryptionKeysTitle": "Luodaan salausavaimia...", @@ -62,21 +70,38 @@ "supportDevs": "Tilaa ente tukeaksesi tätä hanketta.", "supportDiscount": "Käytä kuponkikoodia \"AUTH\" saadaksesi 10% alennuksen ensimmäisestä vuodesta", "changeEmail": "vaihda sähköpostiosoitteesi", + "changePassword": "Vaihda salasana", + "importCodes": "Tuo koodit", + "importTypePlainText": "Pelkkä teksti", + "importTypeEnteEncrypted": "Ente salattu vienti", + "importSelectJsonFile": "Valitse JSON-tiedosto", + "exportCodes": "Vie koodit", + "importLabel": "Tuo", "selectFile": "Valitse tiedosto", + "emailVerificationToggle": "Sähköpostivahvistus", "ok": "Selvä", "cancel": "Keskeytä", "yes": "Kyllä", "no": "Ei", "email": "Sähköposti", "support": "Tuki", + "general": "Yleiset", "settings": "Asetukset", "copied": "Jäljennetty", "pleaseTryAgain": "Yritä uudestaan", "existingUser": "Jo valmiiksi olemassaoleva käyttäjä", + "newUser": "Uusi Ente-käyttäjä", "delete": "Poista", "enterYourPasswordHint": "Syötä salasanasi", "forgotPassword": "Olen unohtanut salasanani", "oops": "Hupsista", + "suggestFeatures": "Ehdota parannuksia", + "faq": "Usein kysyttyä", + "faq_q_1": "Kuinka turvallinen Auth on?", + "faq_q_3": "Kuinka voin poistaa koodeja?", + "faq_q_4": "Kuinka voin tukea tätä projektia?", + "faq_q_5": "Miten voin ottaa käyttöön FaceID-lukituksen Authissa", + "faq_a_5": "Voit ottaa FaceID-lukituksen käyttöön kohdassa Asetukset → Turvallisuus → Lukitusnäyttö.", "somethingWentWrongMessage": "Jokin meni pieleen, yritä uudelleen", "leaveFamily": "Poistu perheestä", "leaveFamilyMessage": "Oletko varma että haluat jättää tämän perhemallin?", @@ -85,9 +110,12 @@ "scan": "Lue", "scanACode": "Lue koodi", "verify": "Vahvista", + "verifyEmail": "Vahvista sähköpostiosoite", "enterCodeHint": "Syötä 6-merkkinen koodi varmennussovelluksestasi", "lostDeviceTitle": "Kadonnut laite?", "twoFactorAuthTitle": "Kaksivaiheinen vahvistus", + "passkeyAuthTitle": "Avainkoodin vahvistus", + "verifyPasskey": "Vahvista avainkoodi", "recoverAccount": "Palauta tilisi", "enterRecoveryKeyHint": "Syötä palautusavaimesi", "recover": "Palauta", @@ -99,6 +127,7 @@ } } }, + "invalidQRCode": "Virheellinen QR-koodi", "noRecoveryKeyTitle": "Ei palautusavainta?", "enterEmailHint": "Syötä sähköpostiosoitteesi", "invalidEmailTitle": "Epäkelpo sähköpostiosoite", @@ -116,7 +145,65 @@ "confirmPassword": "Vahvista salasana", "close": "Sulje", "oopsSomethingWentWrong": "Hupsista! Jotakin meni nyt pieleen.", + "selectLanguage": "Valitse kieli", + "language": "Kieli", + "security": "Turvallisuus", + "lockscreen": "Lukitusnäyttö", + "lockScreenEnablePreSteps": "Aktivoi lukitusnäyttö ottamalla käyttöön laitteen salasana tai näytön lukitus järjestelmäasetuksissa.", + "searchHint": "Etsi...", + "search": "Etsi", + "noResult": "Ei tuloksia", + "addCode": "Lisää koodi", + "enterDetailsManually": "Syötä tiedot manuaalisesti", + "edit": "Muokkaa", + "copiedToClipboard": "Kopioitu leikepöydälle", + "copiedNextToClipboard": "Seuraava koodi kopioitu leikepöydälle", + "error": "Virhe", + "recoveryKeyCopiedToClipboard": "Palautusavain kopioitu leikepöydälle", + "recoveryKeyOnForgotPassword": "Jos unohdat salasanasi, ainoa tapa palauttaa tietosi on tällä avaimella.", + "recoveryKeySaveDescription": "Emme tallenna tätä avainta, ole hyvä ja tallenna tämä 24 sanan avain turvalliseen paikkaan.", + "doThisLater": "Tee tämä myöhemmin", + "saveKey": "Tallenna avain", + "save": "Tallenna", + "send": "Lähetä", + "back": "Takaisin", + "createAccount": "Luo tili", + "password": "Salasana", + "encryption": "Salaus", + "setPasswordTitle": "Luo salasana", + "changePasswordTitle": "Vaihda salasana", + "resetPasswordTitle": "Nollaa salasana", + "encryptionKeys": "Salausavaimet", + "passwordChangedSuccessfully": "Salasana vaihdettu onnistuneesti", + "generatingEncryptionKeys": "Luodaan salausavaimia...", + "continueLabel": "Jatka", + "logInLabel": "Kirjaudu sisään", + "logout": "Kirjaudu ulos", + "yesLogout": "Kyllä, kirjaudu ulos", + "exit": "Poistu", + "about": "Tietoa", + "weAreOpenSource": "Olemme avoimen lähdekoodin ohjelma!", + "privacy": "Yksityisyys", + "checkForUpdates": "Tarkista päivitykset", + "downloadUpdate": "Lataa", + "updateAvailable": "Päivitys saatavilla", + "warning": "Varoitus", + "enterPassword": "Syötä salasana", + "singIn": "Kirjaudu sisään", + "shouldHideCode": "Piilota koodit", "editCodeAuthMessage": "Autentikoidu muokataksesi koodia", "deleteCodeAuthMessage": "Autentikoidu poistaaksesi koodin", - "showQRAuthMessage": "Autentikoidu näyttääksesi QR-koodin" + "showQRAuthMessage": "Autentikoidu näyttääksesi QR-koodin", + "androidBiometricSuccess": "Kirjautuminen onnistui", + "@androidBiometricSuccess": { + "description": "Message to let the user know that authentication was successful. It is used on Android side. Maximum 60 characters." + }, + "androidCancelButton": "Peruuta", + "@androidCancelButton": { + "description": "Message showed on a button that the user can click to leave the current dialog. It is used on Android side. Maximum 30 characters." + }, + "goToSettings": "Mene asetuksiin", + "@goToSettings": { + "description": "Message showed on a button that the user can click to go to settings pages from the current dialog. It is used on both Android and iOS side. Maximum 30 characters." + } } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_fr.arb b/auth/lib/l10n/arb/app_fr.arb index ab18a61559..bddd4982c4 100644 --- a/auth/lib/l10n/arb/app_fr.arb +++ b/auth/lib/l10n/arb/app_fr.arb @@ -263,12 +263,15 @@ "exportLogs": "Exporter les journaux", "enterYourRecoveryKey": "Entrez votre clé de récupération", "tempErrorContactSupportIfPersists": "Il semble qu'une erreur s'est produite. Veuillez réessayer après un certain temps. Si l'erreur persiste, veuillez contacter notre équipe d'assistance.", + "networkHostLookUpErr": "Impossible de se connecter à Ente, veuillez vérifier vos paramètres réseau et contacter le support si l'erreur persiste.", + "networkConnectionRefusedErr": "Impossible de se connecter à Ente, veuillez réessayer après un certain temps. Si l'erreur persiste, veuillez contacter le support.", "itLooksLikeSomethingWentWrongPleaseRetryAfterSome": "Il semble qu'une erreur s'est produite. Veuillez réessayer après un certain temps. Si l'erreur persiste, veuillez contacter notre équipe d'assistance.", "about": "À propos", "weAreOpenSource": "Nous sommes open source !", "privacy": "Confidentialité", "terms": "Conditions", "checkForUpdates": "Vérifier les mises à jour", + "checkStatus": "Vérifier le statut", "downloadUpdate": "Télécharger", "criticalUpdateAvailable": "Mise à jour critique disponible", "updateAvailable": "Une mise à jour est disponible", @@ -417,6 +420,9 @@ "waitingForBrowserRequest": "En attente de la requête du navigateur...", "waitingForVerification": "En attente de vérification...", "passkey": "Code d'accès", + "passKeyPendingVerification": "La vérification est toujours en attente", + "loginSessionExpired": "Session expirée", + "loginSessionExpiredDetails": "Votre session a expiré. Veuillez vous reconnecter.", "developerSettingsWarning": "Êtes-vous sûr de vouloir modifier les paramètres du développeur ?", "developerSettings": "Paramètres du développeur", "serverEndpoint": "Point de terminaison serveur", @@ -439,5 +445,19 @@ "updateNotAvailable": "Mise à jour non disponible", "viewRawCodes": "Afficher les codes bruts", "rawCodes": "Codes bruts", - "rawCodeData": "Données de code brut" + "rawCodeData": "Données de code brut", + "noSystemLockFound": "Aucun verrou système trouvé", + "autoLock": "Verrouillage automatique", + "immediately": "Immédiatement", + "reEnterPassword": "Ressaisir le mot de passe", + "reEnterPin": "Ressaisir le code PIN", + "next": "Suivant", + "tooManyIncorrectAttempts": "Trop de tentatives incorrectes", + "tapToUnlock": "Appuyer pour déverrouiller", + "setNewPassword": "Définir un nouveau mot de passe", + "deviceLock": "Verrouillage de l'appareil", + "hideContent": "Masquer le contenu", + "enterPin": "Saisir le code PIN", + "setNewPin": "Définir un nouveau code PIN", + "importFailureDescNew": "Impossible de lire le fichier sélectionné." } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_it.arb b/auth/lib/l10n/arb/app_it.arb index 0170ecdac1..a0a1f04523 100644 --- a/auth/lib/l10n/arb/app_it.arb +++ b/auth/lib/l10n/arb/app_it.arb @@ -465,5 +465,6 @@ "appLockDescription": "Scegli tra la schermata di blocco predefinita del dispositivo e una schermata di blocco personalizzata con PIN o password.", "pinLock": "Blocco con PIN", "enterPin": "Inserisci PIN", - "setNewPin": "Imposta un nuovo PIN" + "setNewPin": "Imposta un nuovo PIN", + "importFailureDescNew": "Impossibile elaborare il file selezionato." } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_nl.arb b/auth/lib/l10n/arb/app_nl.arb index 6b40e677b9..d0ffb56a20 100644 --- a/auth/lib/l10n/arb/app_nl.arb +++ b/auth/lib/l10n/arb/app_nl.arb @@ -442,5 +442,28 @@ "deleteTagTitle": "Label verwijderen?", "deleteTagMessage": "Weet je zeker dat je deze label wilt verwijderen? Deze actie is onomkeerbaar.", "somethingWentWrongParsingCode": "We konden {x} codes niet verwerken.", - "updateNotAvailable": "Update niet beschikbaar" + "updateNotAvailable": "Update niet beschikbaar", + "viewRawCodes": "Ruwe codes weergeven", + "rawCodes": "Ruwe codes", + "rawCodeData": "Onbewerkte code gegevens", + "appLock": "App-vergrendeling", + "noSystemLockFound": "Geen systeemvergrendeling gevonden", + "toEnableAppLockPleaseSetupDevicePasscodeOrScreen": "Om schermvergrendeling in te schakelen, moet u een toegangscode of schermvergrendeling instellen in uw systeeminstellingen.", + "autoLock": "Automatische vergrendeling", + "immediately": "Onmiddellijk", + "reEnterPassword": "Wachtwoord opnieuw invoeren", + "reEnterPin": "PIN opnieuw invoeren", + "next": "Volgende", + "tooManyIncorrectAttempts": "Te veel onjuiste pogingen", + "tapToUnlock": "Tik om te ontgrendelen", + "setNewPassword": "Nieuw wachtwoord instellen", + "deviceLock": "Apparaat vergrendeling", + "hideContent": "Inhoud verbergen", + "hideContentDescriptionAndroid": "Verbergt de app inhoud in de app switcher en schakelt schermafbeeldingen uit", + "hideContentDescriptioniOS": "Verbergt de inhoud van de app in de app switcher", + "autoLockFeatureDescription": "Tijd waarna de app vergrendelt nadat ze op de achtergrond is gezet", + "appLockDescription": "Kies tussen de standaard schermvergrendeling van uw apparaat en een aangepaste schermvergrendeling met een pincode of wachtwoord.", + "pinLock": "Pin vergrendeling", + "enterPin": "Pin invoeren", + "setNewPin": "Nieuwe pin instellen" } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_pl.arb b/auth/lib/l10n/arb/app_pl.arb index 6b9b390ede..d7ad3c4f1f 100644 --- a/auth/lib/l10n/arb/app_pl.arb +++ b/auth/lib/l10n/arb/app_pl.arb @@ -83,7 +83,7 @@ "passwordForDecryptingExport": "Hasło do odszyfrowania eksportu", "passwordEmptyError": "Pole hasło nie może być puste", "importFromApp": "Importuj kody z {appName}", - "importGoogleAuthGuide": "Wyeksportuj twoje konta z Google Authenticator do kodu QR używając opcji \"Przenieś konta\". Potem używając innego urządzenia, zeskanuj kod QR.", + "importGoogleAuthGuide": "Wyeksportuj Twoje konta z Google Authenticator do kodu QR używając opcji \"Przenieś konta\". Potem używając innego urządzenia, zeskanuj kod QR.\n\nWskazówka: Możesz użyć kamery Twojego laptopa, by zrobić zdjęcie kodu QR.", "importSelectJsonFile": "Wybierz plik JSON", "importSelectAppExport": "Wybierz plik eksportu {appName}", "importEnteEncGuide": "Wybierz zaszyfrowany plik JSON wyeksportowany z Ente", @@ -465,5 +465,6 @@ "appLockDescription": "Wybierz między domyślnym ekranem blokady urządzenia a niestandardowym ekranem blokady z kodem PIN lub hasłem.", "pinLock": "Blokada PIN", "enterPin": "Wprowadź kod PIN", - "setNewPin": "Ustaw nowy kod PIN" + "setNewPin": "Ustaw nowy kod PIN", + "importFailureDescNew": "Nie udało się przetworzyć wybranego pliku." } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_pt.arb b/auth/lib/l10n/arb/app_pt.arb index e252721a46..e9bb5f06c4 100644 --- a/auth/lib/l10n/arb/app_pt.arb +++ b/auth/lib/l10n/arb/app_pt.arb @@ -465,5 +465,6 @@ "appLockDescription": "Escolha entre a tela de bloqueio padrão do seu dispositivo e uma tela de bloqueio personalizada com PIN ou senha.", "pinLock": "Bloqueio PIN", "enterPin": "Insira o PIN", - "setNewPin": "Definir novo PIN" + "setNewPin": "Definir novo PIN", + "importFailureDescNew": "Não foi possível analisar o arquivo selecionado." } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_ro.arb b/auth/lib/l10n/arb/app_ro.arb index cc8cdd1545..9f7c583bda 100644 --- a/auth/lib/l10n/arb/app_ro.arb +++ b/auth/lib/l10n/arb/app_ro.arb @@ -61,6 +61,7 @@ "recreatePassword": "Recreează parola", "incorrectPasswordTitle": "Parolă incorectă", "welcomeBack": "Bine ai revenit!", + "madeWithLoveAtPrefix": "creat cu ❤️ la ", "supportDevs": "Abonează-te la ente pentru a ne susține", "supportDiscount": "Folosește codul \"AUTH\" pentru a obține o reducere de 10% în primul an", "changeEmail": "Schimbă e-mailul", @@ -68,9 +69,11 @@ "data": "Date", "importCodes": "Importă coduri", "importTypePlainText": "Text simplu", + "importTypeEnteEncrypted": "Export Ente criptat", "passwordForDecryptingExport": "Parola pentru a decripta exportul", "passwordEmptyError": "Parola nu poate fi goală", "importFromApp": "Importă coduri din {appName}", + "importGoogleAuthGuide": "Exportă-ți conturile din Google Autheticator cu un cod QR utilizând opțiunea „Transfer conturi”. Apoi, utilizând alt dispozitiv, scanați codul QR.\n\nSfat: Poți utiliza camera web a laptopul-ui pentru a scana codul QR.", "importSelectJsonFile": "Selectează fișierul JSON", "importSelectAppExport": "Selectează fișierul de export din {appName}", "importEnteEncGuide": "Selectează fișierul criptat JSON exportat din Bențe", @@ -127,11 +130,15 @@ "social": "Social", "security": "Securitate", "lockscreen": "Ecran de blocare", + "search": "Căutare", "scanAQrCode": "Scanează un cod QR", + "edit": "Editare", "copiedToClipboard": "Copiat în clipboard", "copiedNextToClipboard": "Codul următor a fost copiat în clipboard", "error": "Eroare", + "recoveryKeyCopiedToClipboard": "Cheie de recuperare salvată în clipboard", "recoveryKeyOnForgotPassword": "Dacă îți uiți parola, singura modalitate prin care poți recupera datele este cu această cheie.", + "recoveryKeySaveDescription": "Nu stocăm această cheie, vă rugăm salvați această cheie de 24 de cuvinte într-un loc sigur.", "saveKey": "Salvare cheie", "save": "Salvare", "send": "Trimitere", diff --git a/auth/lib/l10n/arb/app_sk.arb b/auth/lib/l10n/arb/app_sk.arb index b32cdda7d3..faf3913ac9 100644 --- a/auth/lib/l10n/arb/app_sk.arb +++ b/auth/lib/l10n/arb/app_sk.arb @@ -6,14 +6,21 @@ "@counterAppBarTitle": { "description": "Text shown in the AppBar of the Counter Page" }, + "onBoardingBody": "Zabezpečte svoje kódy 2FA", "onBoardingGetStarted": "Poďme na to", + "setupFirstAccount": "Vytvorte svoj prvý účet", "importScanQrCode": "Naskenovať QR kód", "qrCode": "QR kód", + "importEnterSetupKey": "Vložte kľúč nastavenia", + "importAccountPageTitle": "Vložte detaily o konte", + "secretCanNotBeEmpty": "Tajný kľúč nemôže ostať prázdny", + "bothIssuerAndAccountCanNotBeEmpty": "Buď vydavateľ alebo účet nemôže ostať prázdny", "incorrectDetails": "Chybné údaje", "pleaseVerifyDetails": "Prosím, skontrolujte svoje údaje a skúste to znova", "codeIssuerHint": "Vydavateľ", "codeSecretKeyHint": "Tajný kľúč", "codeAccountHint": "Konto (ucet@domena.com)", + "codeTagHint": "Tag", "accountKeyType": "Typ kľúča", "sessionExpired": "Relácia vypršala", "@sessionExpired": { @@ -44,16 +51,28 @@ "reportABug": "Nahlásiť chybu", "crashAndErrorReporting": "Hlásenie zlyhaní a chýb", "reportBug": "Nahlásiť chybu", + "emailUsMessage": "Pošlite nám email na adresu {email}", + "@emailUsMessage": { + "placeholders": { + "email": { + "type": "String" + } + } + }, "contactSupport": "Kontaktovať podporu", "rateUsOnStore": "Ohodnoťte nás cez {storeName}", "blog": "Blog", "merchandise": "Merchandise", + "verifyPassword": "Potvrďte heslo", "pleaseWait": "Prosím počkajte...", "generatingEncryptionKeysTitle": "Generovanie šifrovacích kľúčov...", + "recreatePassword": "Resetovať heslo", + "recreatePasswordMessage": "Aktuálne zariadenie nie je dostatočne výkonné na overenie vášho hesla, takže ho musíme regenerovať raz spôsobom, ktorý funguje vo všetkých zariadeniach.\n\nPrihláste sa pomocou kľúča na obnovenie a znovu vygenerujte svoje heslo (ak si prajete, môžete znova použiť rovnaké).", "useRecoveryKey": "Použiť kľúč na obnovenie", "incorrectPasswordTitle": "Nesprávne heslo", "welcomeBack": "Vitajte späť!", "madeWithLoveAtPrefix": "vyrobené so ❤️ v ", + "supportDevs": "Predplaďte si ente a podporte nás", "supportDiscount": "Použite kód \"AUTH\" pre získanie 10% zľavy na prvý rok", "changeEmail": "Zmeniť e-mail", "changePassword": "Zmeniť heslo", @@ -75,8 +94,17 @@ "importLastpassGuide": "Použite možnosť \"Preniesť účty\" v nastaveniach služby Lastpass Authenticator a stlačte \"Exportovať účty do súboru\". Importujte stiahnutý JSON súbor.", "exportCodes": "Exportovať kódy", "importLabel": "Importovať", + "importInstruction": "Vyberte súbor, ktorý obsahuje zoznam vašich kódov v nasledujúcom formáte", + "importCodeDelimiterInfo": "Kódy môžu byť oddelené čiarkou alebo novým riadkom", "selectFile": "Vybrať súbor", - "emailVerificationToggle": "Overenie e-mailovej adresy", + "emailVerificationToggle": "Overenie pomocou e-mailovej adresy", + "emailVerificationEnableWarning": "Aby ste predišli vymknutiu sa z vášho účtu, nezabudnite pred povolením overenia emailom uložiť kópiu svojho 2FA emailu mimo Ente Auth.", + "authToChangeEmailVerificationSetting": "Pre zmenu overenia pomocou emailu sa musíte overiť", + "authToViewYourRecoveryKey": "Pre zobrazenie vášho kľúča na obnovenie sa musíte overiť", + "authToChangeYourEmail": "Pre zmenu vášho emailu sa musíte overiť", + "authToChangeYourPassword": "Pre zmenu vášho hesla sa musíte overiť", + "authToViewSecrets": "Pre zobrazenie vašich tajných údajov sa musíte overiť", + "authToInitiateSignIn": "Pre iniciáciu prihlásenia sa pre zálohu sa musíte overiť.", "ok": "Ok", "cancel": "Zrušiť", "yes": "Áno", @@ -104,13 +132,20 @@ "faq_q_4": "Ako môžem podporiť tento projekt?", "faq_a_4": "Vývoj tohto projektu môžete podporiť zakúpením predplatného našej aplikácie Photos na ente.io.", "faq_q_5": "Ako môžem nastaviť FaceID v Auth?", + "faq_a_5": "Zámok FaceID môžete povoliť v sekcii Nastavenia → Zabezpečenie → Uzamknutie obrazovky.", + "somethingWentWrongMessage": "Niečo sa pokazilo, skúste to prosím znova", "leaveFamily": "Opustiť rodinku", "leaveFamilyMessage": "Ste si istý, že chcete opustiť rodinku?", "inFamilyPlanMessage": "Ste prihlásený k rodinke!", + "swipeHint": "Potiahnite doľava pre upravenie alebo vymazanie kódov", "scan": "Skenovať", "scanACode": "Skenovať kód", "verify": "Overiť", "verifyEmail": "Overiť email", + "enterCodeHint": "Zadajte 6-miestny kód z\nvašej overovacej aplikácie", + "lostDeviceTitle": "Stratené zariadenie?", + "twoFactorAuthTitle": "Dvojfaktorové overovanie", + "passkeyAuthTitle": "Overenie pomocou passkey", "verifyPasskey": "Overiť passkey", "recoverAccount": "Obnoviť konto", "enterRecoveryKeyHint": "Vložte váš kód pre obnovenie", @@ -145,8 +180,8 @@ "language": "Jazyk", "social": "Sociálne siete", "security": "Zabezpečenie", - "lockscreen": "Uzamknutá obrazovka", - "authToChangeLockscreenSetting": "Pre zmenu nastavenia uzamknutej obrazovky sa musíte overiť", + "lockscreen": "Uzamknutie obrazovky", + "authToChangeLockscreenSetting": "Pre zmenu nastavenia uzamknutia obrazovky sa musíte overiť", "lockScreenEnablePreSteps": "Pre povolenie uzamknutia obrazovky, nastavte prístupový kód zariadenia alebo zámok obrazovky v nastaveniach systému.", "viewActiveSessions": "Zobraziť aktívne relácie", "authToViewYourActiveSessions": "Pre zobrazenie vašich aktívnych relácii sa musíte overiť", @@ -168,10 +203,73 @@ "saveKey": "Uložiť kľúč", "save": "Uložiť", "send": "Odoslať", + "saveOrSendDescription": "Chcete to uložiť do svojho zariadenia (predvolený priečinok Stiahnuté súbory) alebo to odoslať do iných aplikácií?", + "saveOnlyDescription": "Chcete to uložiť do svojho zariadenia (predvolený priečinok Stiahnuté súbory)?", + "back": "Späť", + "createAccount": "Vytvoriť účet", + "passwordStrength": "Sila hesla: {passwordStrengthValue}", + "@passwordStrength": { + "description": "Text to indicate the password strength", + "placeholders": { + "passwordStrengthValue": { + "description": "The strength of the password as a string", + "type": "String", + "example": "Weak or Moderate or Strong" + } + }, + "message": "Password Strength: {passwordStrengthText}" + }, + "password": "Heslo", + "signUpTerms": "Súhlasím s podmienkami používania a zásadami ochrany osobných údajov", + "privacyPolicyTitle": "Zásady ochrany osobných údajov", + "termsOfServicesTitle": "Podmienky používania", + "encryption": "Šifrovanie", + "setPasswordTitle": "Nastaviť heslo", + "changePasswordTitle": "Zmeniť heslo", + "resetPasswordTitle": "Obnoviť heslo", + "encryptionKeys": "Šifrovacie kľúče", + "passwordWarning": "Ente neukladá tohto heslo. V prípade, že ho zabudnete, nie sme schopní rozšifrovať vaše údaje", + "enterPasswordToEncrypt": "Zadajte heslo, ktoré môžeme použiť na šifrovanie vašich údajov", + "enterNewPasswordToEncrypt": "Zadajte nové heslo, ktoré môžeme použiť na šifrovanie vašich údajov", + "passwordChangedSuccessfully": "Heslo bolo úspešne zmenené", + "generatingEncryptionKeys": "Generovanie šifrovacích kľúčov...", + "continueLabel": "Pokračovať", + "insecureDevice": "Slabo zabezpečené zariadenie", + "sorryWeCouldNotGenerateSecureKeysOnThisDevicennplease": "Ospravedlňujeme sa, v tomto zariadení sme nemohli generovať bezpečnostné kľúče.\n\nzaregistrujte sa z iného zariadenia.", + "howItWorks": "Ako to funguje", + "ackPasswordLostWarning": "Rozumiem, že ak stratím alebo zabudnem heslo, môžem stratiť svoje údaje, pretože moje údaje sú šifrované end-to-end.", + "loginTerms": "Kliknutím na prihlásenie, súhlasím s podmienkami používania a zásadami ochrany osobných údajov", + "logInLabel": "Prihlásenie", + "logout": "Odhlasenie", + "areYouSureYouWantToLogout": "Naozaj sa chcete odhlásiť?", + "yesLogout": "Áno, odhlásiť sa", + "exit": "Ukončiť", + "verifyingRecoveryKey": "Overovanie kľúča na obnovenie...", + "recoveryKeyVerified": "Kľúč na obnovenie overený", + "recoveryKeySuccessBody": "Skvelé! Váš kľúč na obnovenie je správny. Ďakujeme za overenie.\n\nNezabudnite uchovať váš kľúč na obnovenie uložený bezpečne.", + "invalidRecoveryKey": "Zadaný kľúč na obnovenie nie je platný. Uistite sa, že obsahuje 24 slov a skontrolujte písmenko po písmenku každé z nich.\n\nAk ste zadali starší kód na obnovenie, uistite sa, že je dlhý 64 znakov a skontrolujte každý znak samostatne.", + "recreatePasswordTitle": "Resetovať heslo", + "recreatePasswordBody": "Aktuálne zariadenie nie je dostatočne výkonné na overenie vášho hesla, avšak vieme ho regenerovať spôsobom, ktorý funguje vo všetkých zariadeniach.\n\nPrihláste sa pomocou kľúča na obnovenie a znovu vygenerujte svoje heslo (ak si prajete, môžete znova použiť rovnaké).", + "invalidKey": "Neplatný kľúč", + "tryAgain": "Skúsiť znova", + "viewRecoveryKey": "Zobraziť kľúč na obnovenie", + "confirmRecoveryKey": "Potvrdiť kód pre obnovenie", + "recoveryKeyVerifyReason": "Váš kľúč na obnovenie je jediný spôsob, ako obnoviť svoje fotografie, ak zabudnete heslo. Kľúč na obnovenie nájdete v Nastavenia > Konto.\n\nZadajte tu svoj kľúč na obnovenie a overte, či ste ho správne uložili.", + "confirmYourRecoveryKey": "Potvrďte váš kód pre obnovenie", + "confirm": "Potvrdiť", + "emailYourLogs": "Odoslať vaše logy emailom", + "pleaseSendTheLogsTo": "Prosím, pošlite logy na adresu \n{toEmail}", + "copyEmailAddress": "Skopírovať e-mailovú adresu", + "exportLogs": "Exportovať logy", + "enterYourRecoveryKey": "Vložte váš kód pre obnovenie", + "tempErrorContactSupportIfPersists": "Vyzerá to, že sa niečo pokazilo. Skúste znova v krátkom čase. Ak chyba pretrváva, kontaktujte náš tím podpory.", + "networkHostLookUpErr": "Nemožno sa pripojiť k Ente, skontrolujte svoje nastavenia siete a kontaktujte podporu, ak chyba pretrváva.", + "networkConnectionRefusedErr": "Nemožno sa pripojiť k Ente, skúste znova v krátkom čase. Ak chyba pretrváva, kontaktujte podporu.", + "itLooksLikeSomethingWentWrongPleaseRetryAfterSome": "Vyzerá to, že sa niečo pokazilo. Skúste znova v krátkom čase. Ak chyba pretrváva, kontaktujte náš tím podpory.", "about": "O aplikácii", "weAreOpenSource": "We are open source!", - "privacy": "Ochrana osobných údajov", - "terms": "Podmienky používania", + "privacy": "Súkromie", + "terms": "Podmienky", "checkForUpdates": "Zistiť dostupnosť aktualizácií", "checkStatus": "Overiť stav", "downloadUpdate": "Stiahnuť", @@ -181,14 +279,192 @@ "checking": "Kontrolovanie...", "youAreOnTheLatestVersion": "Používate najnovšiu verziu", "warning": "Upozornenie", + "exportWarningDesc": "Exportovaný súbor obsahuje citlivé informácie. Prosím, uložte to bezpečne.", "iUnderStand": "Rozumiem", "@iUnderStand": { "description": "Text for the button to confirm the user understands the warning" }, + "authToExportCodes": "Pre export vašich kódov sa musíte overiť", "importSuccessTitle": "Jéj!", + "importSuccessDesc": "Úspešne ste importovali kódy v počte {count}!", + "@importSuccessDesc": { + "placeholders": { + "count": { + "description": "The number of codes imported", + "type": "int", + "example": "1" + } + } + }, "sorry": "Ospravedlňujeme sa", "importFailureDesc": "Vybraný súbor nie je možné spracovať.\nAk potrebujete pomoc, napíšte na adresu support@ente.io!", "pendingSyncs": "Upozornenie", + "pendingSyncsWarningBody": "Niektoré z vašich kódov neboli zálohované.\n\nPred odhlásením sa uistite, že máte zálohu pre tieto kódy.", + "checkInboxAndSpamFolder": "Skontrolujte svoju doručenú poštu (a spam) pre dokončenie overenia", "tapToEnterCode": "Klepnutím zadajte kód", - "resendEmail": "Znovu odoslať email" + "resendEmail": "Znovu odoslať email", + "weHaveSendEmailTo": "Odoslali sme email na adresu {email}", + "@weHaveSendEmailTo": { + "description": "Text to indicate that we have sent a mail to the user", + "placeholders": { + "email": { + "description": "The email address of the user", + "type": "String", + "example": "example@ente.io" + } + } + }, + "activeSessions": "Aktívne relácie", + "somethingWentWrongPleaseTryAgain": "Niečo sa pokazilo, skúste to prosím znova", + "thisWillLogYouOutOfThisDevice": "Toto vás odhlási z tohto zariadenia!", + "thisWillLogYouOutOfTheFollowingDevice": "Toto vás odhlási z následujúceho zariadenia:", + "terminateSession": "Ukončiť reláciu?", + "terminate": "Ukončiť", + "thisDevice": "Toto zariadenie", + "toResetVerifyEmail": "Ak chcete obnoviť svoje heslo, najskôr overte svoj email.", + "thisEmailIsAlreadyInUse": "Tento e-mail sa už používa", + "verificationFailedPleaseTryAgain": "Overenie zlyhalo, skúste to znova", + "yourVerificationCodeHasExpired": "Platnosť overovacieho kódu uplynula", + "incorrectCode": "Neplatný kód", + "sorryTheCodeYouveEnteredIsIncorrect": "Ľutujeme, zadaný kód je nesprávny", + "emailChangedTo": "Emailová adresa bola zmenená na {newEmail}", + "authenticationFailedPleaseTryAgain": "Overenie zlyhalo. Skúste to znova", + "authenticationSuccessful": "Overenie sa podarilo!", + "twofactorAuthenticationSuccessfullyReset": "Dvojfaktorové overovanie bolo úspešne obnovené", + "incorrectRecoveryKey": "Nesprávny kľúč na obnovenie", + "theRecoveryKeyYouEnteredIsIncorrect": "Kľúč na obnovenie, ktorý ste zadali, je nesprávny", + "enterPassword": "Zadajte heslo", + "selectExportFormat": "Zvoľte formát pre exportovanie", + "exportDialogDesc": "Šifrované exporty budú chránené heslom, ktoré si vyberiete.", + "encrypted": "Šifrované", + "plainText": "Obyčajný text", + "passwordToEncryptExport": "Heslo na zašifrovanie exportu", + "export": "Exportovať", + "useOffline": "Používať bez zálohy", + "signInToBackup": "Prihláste sa a zálohujte svoje kódy", + "singIn": "Prihlásiť sa", + "sigInBackupReminder": "Exportujte svoje kódy, aby ste sa uistili, že máte zálohu, ktorú môžete neskôr obnoviť.", + "offlineModeWarning": "Rozhodli ste sa pokračovať bez zálohovania. Prosím, vykonávajte pravidelné manuálne zálohy aby ste mali istotu, že kódy nestratíte.", + "showLargeIcons": "Zobraziť veľké ikony", + "shouldHideCode": "Skryť kódy", + "doubleTapToViewHiddenCode": "Dvakrát klepnite na položku aby ste zobrazili kód", + "focusOnSearchBar": "Využívať pole vyhľadávania pri spustení aplikácie", + "confirmUpdatingkey": "Ste si istí, že chcete zmeniť tajný kľúč?", + "minimizeAppOnCopy": "Minimalizovať po skopírovaní", + "editCodeAuthMessage": "Overte sa pre zmenu kódu", + "deleteCodeAuthMessage": "Overte sa pre vymazanie kódu", + "showQRAuthMessage": "Overte sa pre zobrazenie QR kódu", + "confirmAccountDeleteTitle": "Potvrdiť odstránenie účtu", + "confirmAccountDeleteMessage": "Tento účet je prepojený s inými aplikáciami Ente, ak nejaké používate.\n\nVšetky nahrané údaje v aplikáciách od Ente budú naplánované na výmaz a váš účet bude natrvalo odstránený.", + "androidBiometricHint": "Overiť identitu", + "@androidBiometricHint": { + "description": "Hint message advising the user how to authenticate with biometrics. It is used on Android side. Maximum 60 characters." + }, + "androidBiometricNotRecognized": "Nerozpoznané. Skúste znova.", + "@androidBiometricNotRecognized": { + "description": "Message to let the user know that authentication was failed. It is used on Android side. Maximum 60 characters." + }, + "androidBiometricSuccess": "Overenie úspešné", + "@androidBiometricSuccess": { + "description": "Message to let the user know that authentication was successful. It is used on Android side. Maximum 60 characters." + }, + "androidCancelButton": "Zrušiť", + "@androidCancelButton": { + "description": "Message showed on a button that the user can click to leave the current dialog. It is used on Android side. Maximum 30 characters." + }, + "androidSignInTitle": "Vyžaduje sa overenie", + "@androidSignInTitle": { + "description": "Message showed as a title in a dialog which indicates the user that they need to scan biometric to continue. It is used on Android side. Maximum 60 characters." + }, + "androidBiometricRequiredTitle": "Vyžaduje sa biometria", + "@androidBiometricRequiredTitle": { + "description": "Message showed as a title in a dialog which indicates the user has not set up biometric authentication on their device. It is used on Android side. Maximum 60 characters." + }, + "androidDeviceCredentialsRequiredTitle": "Vyžadujú sa poverenia zariadenia", + "@androidDeviceCredentialsRequiredTitle": { + "description": "Message showed as a title in a dialog which indicates the user has not set up credentials authentication on their device. It is used on Android side. Maximum 60 characters." + }, + "androidDeviceCredentialsSetupDescription": "Vyžadujú sa poverenia zariadenia", + "@androidDeviceCredentialsSetupDescription": { + "description": "Message advising the user to go to the settings and configure device credentials on their device. It shows in a dialog on Android side." + }, + "goToSettings": "Prejsť do nastavení", + "@goToSettings": { + "description": "Message showed on a button that the user can click to go to settings pages from the current dialog. It is used on both Android and iOS side. Maximum 30 characters." + }, + "androidGoToSettingsDescription": "Overenie pomocou biometrie nie je na vašom zariadení nastavené. Prejdite na 'Nastavenie > Zabezpečenie' a pridajte overenie pomocou biometrie.", + "@androidGoToSettingsDescription": { + "description": "Message advising the user to go to the settings and configure biometric on their device. It shows in a dialog on Android side." + }, + "iOSLockOut": "Overenie pomocou biometrie je zakázané. Zamknite a odomknite svoju obrazovku, aby ste ho povolili.", + "@iOSLockOut": { + "description": "Message advising the user to re-enable biometrics on their device. It shows in a dialog on iOS side." + }, + "iOSGoToSettingsDescription": "Overenie pomocou biometrie nie je na vašom zariadení nastavené. Povoľte buď Touch ID or Face ID na svojom telefóne.", + "@iOSGoToSettingsDescription": { + "description": "Message advising the user to go to the settings and configure Biometrics for their device. It shows in a dialog on iOS side." + }, + "iOSOkButton": "OK", + "@iOSOkButton": { + "description": "Message showed on a button that the user can click to leave the current dialog. It is used on iOS side. Maximum 30 characters." + }, + "noInternetConnection": "Žiadne internetové pripojenie", + "pleaseCheckYourInternetConnectionAndTryAgain": "Skontrolujte svoje internetové pripojenie a skúste to znova.", + "signOutFromOtherDevices": "Odhlásiť sa z iných zariadení", + "signOutOtherBody": "Ak si myslíte, že niekto môže vedieť vaše heslo, môžete vynútiť odhlásenie všetkých ostatných zariadení vo vašom účte.", + "signOutOtherDevices": "Odhlásiť iné zariadenie", + "doNotSignOut": "Neodhlasovať", + "hearUsWhereTitle": "Ako ste sa dozvedeli o Ente? (voliteľné)", + "hearUsExplanation": "Nesledujeme inštalácie aplikácie. Veľmi by nám pomohlo, keby ste nám povedali, ako ste sa o nás dozvedeli!", + "recoveryKeySaved": "Kľúč na obnovenie uložený v priečinku Stiahnutých súborov!", + "waitingForBrowserRequest": "Čakanie na prehliadač...", + "waitingForVerification": "Čakanie na overenie...", + "passkey": "Passkey", + "passKeyPendingVerification": "Overenie stále prebieha", + "loginSessionExpired": "Relácia vypršala", + "loginSessionExpiredDetails": "Vaša relácia vypršala. Prosím, prihláste sa znovu.", + "developerSettingsWarning": "Ste si istí, že chcete modifikovať nastavenia pre vývojárov?", + "developerSettings": "Nastavenia pre vývojárov", + "serverEndpoint": "Endpoint servera", + "invalidEndpoint": "Neplatný endpoint", + "invalidEndpointMessage": "Ospravedlňujeme sa, endpoint, ktorý ste zadali, je neplatný. Zadajte platný endpoint a skúste to znova.", + "endpointUpdatedMessage": "Endpoint úspešne aktualizovaný", + "customEndpoint": "Pripojený k endpointu {endpoint}", + "pinText": "Pripnúť", + "unpinText": "Odopnúť", + "pinnedCodeMessage": "{code} bol pripnutý", + "unpinnedCodeMessage": "{code} bol odopnutý", + "tags": "Tagy", + "createNewTag": "Vytvoriť nový tag", + "tag": "Tag", + "create": "Vytvoriť", + "editTag": "Upraviť tag", + "deleteTagTitle": "Odstrániť tag?", + "deleteTagMessage": "Naozaj chcete odstrániť tag? Táto akcia je nezvratná.", + "somethingWentWrongParsingCode": "Neboli sme schopní spracovať {x} kódov.", + "updateNotAvailable": "K dispozícii nie je žiadna aktualizácia", + "viewRawCodes": "Zobraziť nešifrované kódy", + "rawCodes": "Nešifrované kódy", + "rawCodeData": "Nešifrované údaje o kódoch", + "appLock": "Zámok aplikácie", + "noSystemLockFound": "Nenájdená žiadna zámka obrazovky", + "toEnableAppLockPleaseSetupDevicePasscodeOrScreen": "Pre povolenie uzamknutia aplikácie, nastavte prístupový kód zariadenia alebo zámok obrazovky v nastaveniach systému.", + "autoLock": "Automatické uzamknutie", + "immediately": "Okamžite", + "reEnterPassword": "Zadajte heslo znova", + "reEnterPin": "Zadajte PIN znova", + "next": "Ďalej", + "tooManyIncorrectAttempts": "Príliš veľa chybných pokusov", + "tapToUnlock": "Ťuknutím odomknete", + "setNewPassword": "Nastaviť nové heslo", + "deviceLock": "Zámok zariadenia", + "hideContent": "Skryť obsah", + "hideContentDescriptionAndroid": "Skrýva obsah v prepínači aplikácii a zakazuje snímky obrazovky", + "hideContentDescriptioniOS": "Skrýva obsah v prepínači aplikácii", + "autoLockFeatureDescription": "Čas, po ktorom sa aplikácia uzamkne po nečinnosti", + "appLockDescription": "Vyberte si medzi predvolenou zámkou obrazovky vášho zariadenia a vlastnou zámkou obrazovky s PIN kódom alebo heslom.", + "pinLock": "Zámok PIN", + "enterPin": "Zadajte PIN", + "setNewPin": "Nastaviť nový PIN", + "importFailureDescNew": "Vybraný súbor nie je možné spracovať." } \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_uk.arb b/auth/lib/l10n/arb/app_uk.arb new file mode 100644 index 0000000000..906729d60d --- /dev/null +++ b/auth/lib/l10n/arb/app_uk.arb @@ -0,0 +1,470 @@ +{ + "account": "Обліковий запис", + "unlock": "Розблокувати", + "recoveryKey": "Ключ відновлення", + "counterAppBarTitle": "Лічильник", + "@counterAppBarTitle": { + "description": "Text shown in the AppBar of the Counter Page" + }, + "onBoardingBody": "Безпечно зробіть резервну копію кодів 2FA", + "onBoardingGetStarted": "Розпочати", + "setupFirstAccount": "Налаштуйте свій перший обліковий запис", + "importScanQrCode": "Відскануйте QR-код", + "qrCode": "QR-код", + "importEnterSetupKey": "Введіть ключ налаштування", + "importAccountPageTitle": "Введіть дані облікового запису", + "secretCanNotBeEmpty": "Секретний ключ не може бути порожнім", + "bothIssuerAndAccountCanNotBeEmpty": "Не може бути пустим як емітент, так і обліковий запис", + "incorrectDetails": "Невірні дані", + "pleaseVerifyDetails": "Будь ласка, перевірте дані та повторіть спробу", + "codeIssuerHint": "Емітент", + "codeSecretKeyHint": "Секретний ключ", + "codeAccountHint": "Обліковий запис (you@domain.com)", + "codeTagHint": "Мітка", + "accountKeyType": "Тип ключа", + "sessionExpired": "Час сеансу минув", + "@sessionExpired": { + "description": "Title of the dialog when the users current session is invalid/expired" + }, + "pleaseLoginAgain": "Будь ласка, увійдіть знову", + "loggingOut": "Вихід із системи...", + "timeBasedKeyType": "На основі часу (TOTP)", + "counterBasedKeyType": "На основі лічильника (HOTP)", + "saveAction": "Зберегти", + "nextTotpTitle": "далі", + "deleteCodeTitle": "Видалити код?", + "deleteCodeMessage": "Ви впевнені, що хочете видалити цей код? Ця дія є незворотною.", + "viewLogsAction": "Переглянути журнали", + "sendLogsDescription": "Це надішле журнали, щоб допомогти нам зневадити проблему. Хоча ми вживаємо запобіжні заходи для того, щоб ніяка чутлива інформація не була переслана, ми рекомендуємо вам переглянути ці журнали, перш ніж поділитися ними.", + "preparingLogsTitle": "Підготовка журналів...", + "emailLogsTitle": "Переслати журнали електронною поштою", + "emailLogsMessage": "Будь ласка, надішліть журнали до електронної пошти {email}", + "@emailLogsMessage": { + "placeholders": { + "email": { + "type": "String" + } + } + }, + "copyEmailAction": "Скопіювати електронну пошту", + "exportLogsAction": "Експортувати журнал", + "reportABug": "Повідомити про помилку", + "crashAndErrorReporting": "Звіт про аварії та помилки", + "reportBug": "Повідомити про помилку", + "emailUsMessage": "Будь ласка, напишіть нам за електронною адресою {email}", + "@emailUsMessage": { + "placeholders": { + "email": { + "type": "String" + } + } + }, + "contactSupport": "Звернутися до служби підтримки", + "rateUsOnStore": "Оцініть нас на {storeName}", + "blog": "Блог", + "merchandise": "Товари", + "verifyPassword": "Підтвердження пароля", + "pleaseWait": "Будь ласка, зачекайте...", + "generatingEncryptionKeysTitle": "Створення ключів шифрування...", + "recreatePassword": "Повторно створити пароль", + "recreatePasswordMessage": "Поточний пристрій не є достатньо потужним для підтвердження пароля, тому ми маємо відновити його таким чином, щоб він працював на всіх пристроях.\n\nБудь ласка, увійдіть за допомогою вашого ключа відновлення і відновіть ваш пароль (ви можете знову використати той самий пароль, якщо бажаєте).", + "useRecoveryKey": "Застосувати ключ відновлення", + "incorrectPasswordTitle": "Невірний пароль", + "welcomeBack": "З поверненням!", + "madeWithLoveAtPrefix": "зроблено з ❤️ в ", + "supportDevs": "Підпишіться на ente, щоб підтримати нас", + "supportDiscount": "Використовуйте купон \"AUTH\", щоб отримати 10% знижки за перший рік", + "changeEmail": "Змінити адресу електронної пошти", + "changePassword": "Змінити пароль", + "data": "Дані", + "importCodes": "Імпортувати коди", + "importTypePlainText": "Звичайний текст", + "importTypeEnteEncrypted": "Зашифрований експорт Ente", + "passwordForDecryptingExport": "Пароль для розшифровки експорту", + "passwordEmptyError": "Пароль не може бути порожнім", + "importFromApp": "Імпортувати коди з {appName}", + "importGoogleAuthGuide": "Експортуйте свої облікові записи з Google Authenticator у QR-код за допомогою опції «Перенести облікові записи». Потім за допомогою іншого пристрою відскануйте QR-код.\n\nПорада: Ви можете сфотографувати QR-код за допомогою вебкамери свого ноутбука.", + "importSelectJsonFile": "Оберіть файл JSON", + "importSelectAppExport": "Виберіть експортований файл {appName}", + "importEnteEncGuide": "Виберіть зашифрований файл JSON, експортований з Ente", + "importRaivoGuide": "Використовуйте опцію «Export OTPs to Zip archive» у налаштуваннях Raivo.\n\nРозпакуйте файл ZIP та імпортуйте файл JSON.", + "importBitwardenGuide": "Використовуйте опцію \"Export vault\" в Bitwarden Tools та імпортуйте незашифрований файл JSON.", + "importAegisGuide": "Скористайтеся опцією \"Export the vault\" у параметрах Aegis.\n\nЯкщо ваше сховище зашифровано, вам потрібно буде ввести пароль сховища для його дешифрування.", + "import2FasGuide": "Використовуйте параметр «Settings->Backup -Export» у 2FAS.\n\nЯкщо ваша резервна копія зашифрована, вам потрібно буде ввести пароль, щоб розшифрувати резервну копію", + "importLastpassGuide": "Скористайтеся опцією «Transfer accounts» в налаштуваннях Lastpass Authenticator і натисніть «Export accounts to file». Імпортуйте завантажений JSON.", + "exportCodes": "Експортувати коди", + "importLabel": "Імпортувати", + "importInstruction": "Будь ласка, виберіть файл, що містить список кодів у наступному форматі", + "importCodeDelimiterInfo": "Коди можуть бути розділені комою або новим рядком", + "selectFile": "Вибрати файл", + "emailVerificationToggle": "Підтвердження адреси електронної пошти", + "emailVerificationEnableWarning": "Щоб уникнути блокування доступу до свого облікового запису, обов’язково збережіть копію двофакторної аутентифікації до своєї електронної пошти за межами Ente Auth, перш ніж увімкнути перевірку електронної пошти.", + "authToChangeEmailVerificationSetting": "Будь ласка, пройдіть аутентифікацію, щоб змінити перевірку адреси електронної пошти", + "authToViewYourRecoveryKey": "Будь ласка, пройдіть аутентифікацію, щоб переглянути ваш ключ відновлення", + "authToChangeYourEmail": "Будь ласка, пройдіть аутентифікацію, щоб змінити адресу електронної пошти", + "authToChangeYourPassword": "Будь ласка, пройдіть аутентифікацію, щоб змінити ваш пароль", + "authToViewSecrets": "Будь ласка, пройдіть аутентифікацію, щоб переглянути ваші секретні коди", + "authToInitiateSignIn": "Будь ласка, пройдіть аутентифікацію, щоб розпочати вхід для резервного копіювання.", + "ok": "Ок", + "cancel": "Скасувати", + "yes": "Так", + "no": "Ні", + "email": "Адреса електронної пошти", + "support": "Служба підтримки", + "general": "Загальні", + "settings": "Налаштування", + "copied": "Скопійовано", + "pleaseTryAgain": "Будь ласка, спробуйте ще раз", + "existingUser": "Існуючий користувач", + "newUser": "Вперше на Ente", + "delete": "Видалити", + "enterYourPasswordHint": "Введіть свій пароль", + "forgotPassword": "Нагадати пароль", + "oops": "От халепа", + "suggestFeatures": "Запропонувати нові функції", + "faq": "Часто Запитувані Питання", + "faq_q_1": "Наскільки безпечним є Auth?", + "faq_a_1": "Всі коди, які ви зберігаєте в Auth, кодуються наскрізним захистом. Це означає, що тільки ви можете отримати доступ до ваших кодів. Наші програми мають відкритий вихідний код, і наша криптографія була перевірена зовнішніми аудиторами.", + "faq_q_2": "Чи я можу отримати доступ до своїх кодів на настільному комп'ютері?", + "faq_a_2": "Ви можете отримати доступ до ваших кодів у веб на auth.ente.io.", + "faq_q_3": "Як я можу видалити коди?", + "faq_a_3": "Ви можете видалити код, провівши пальцем вліво на цьому елементі.", + "faq_q_4": "Як я можу підтримати цей проект?", + "faq_a_4": "Ви можете підтримати розробку цього проекту, підписавшись на наш додаток Photos на ente.io.", + "faq_q_5": "Як я можу активувати розблокування за допомогою FaceID в Auth", + "faq_a_5": "Ви можете активувати розблокування за допомогою FaceID у Налаштування → Безпека → Блокування екрану.", + "somethingWentWrongMessage": "Щось пішло не так, спробуйте, будь ласка, знову", + "leaveFamily": "Залишити сімейний план", + "leaveFamilyMessage": "Ви впевнені, що хочете залишити сімейний план?", + "inFamilyPlanMessage": "Ви знаходитесь на сімейному плані!", + "swipeHint": "Проведіть пальцем вліво, щоб редагувати або видаляти коди", + "scan": "Сканувати", + "scanACode": "Сканувати код", + "verify": "Перевірити", + "verifyEmail": "Підтвердити електронну адресу", + "enterCodeHint": "Введіть нижче шестизначний код із застосунку для автентифікації", + "lostDeviceTitle": "Загубили пристрій?", + "twoFactorAuthTitle": "Двофакторна аутентифікація", + "passkeyAuthTitle": "Перевірка секретного ключа", + "verifyPasskey": "Підтвердження секретного ключа", + "recoverAccount": "Відновити обліковий запис", + "enterRecoveryKeyHint": "Введіть ваш ключ відновлення", + "recover": "Відновлення", + "contactSupportViaEmailMessage": "Будь ласка, надішліть електронну пошту на адресу {email} з вашої зареєстрованої адреси електронної пошти", + "@contactSupportViaEmailMessage": { + "placeholders": { + "email": { + "type": "String" + } + } + }, + "invalidQRCode": "Хибний QR-код", + "noRecoveryKeyTitle": "Немає ключа відновлення?", + "enterEmailHint": "Введіть вашу адресу електронної пошти", + "invalidEmailTitle": "Хибна адреса електронної пошти", + "invalidEmailMessage": "Введіть дійсну адресу електронної пошти.", + "deleteAccount": "Видалити обліковий запис", + "deleteAccountQuery": "Нам дуже шкода, що Ви залишаєте нас. Чи Ви зіткнулися з якоюсь проблемою?", + "yesSendFeedbackAction": "Так, надіслати відгук", + "noDeleteAccountAction": "Ні, видаліть мій обліковий запис", + "initiateAccountDeleteTitle": "Будь ласка, авторизуйтесь, щоб розпочати видалення облікового запису", + "sendEmail": "Надіслати електронного листа", + "createNewAccount": "Створити новий обліковий запис", + "weakStrength": "Слабкий", + "strongStrength": "Надійний", + "moderateStrength": "Помірний", + "confirmPassword": "Підтвердити пароль", + "close": "Закрити", + "oopsSomethingWentWrong": "Ой, лихо. Щось пішло не так.", + "selectLanguage": "Виберіть мову", + "language": "Мова", + "social": "Соціальні мережі", + "security": "Безпека", + "lockscreen": "Екран блокування", + "authToChangeLockscreenSetting": "Будь ласка, авторизуйтесь для зміни налаштувань екрану блокування", + "lockScreenEnablePreSteps": "Для увімкнення екрана блокування, будь ласка, налаштуйте пароль пристрою або блокування екрана в системних налаштуваннях.", + "viewActiveSessions": "Показати активні сеанси", + "authToViewYourActiveSessions": "Будь ласка, пройдіть аутентифікацію, щоб переглянути ваші активні сеанси", + "searchHint": "Пошук...", + "search": "Пошук", + "sorryUnableToGenCode": "Вибачте, не вдалося створити код для {issuerName}", + "noResult": "Немає результатів", + "addCode": "Додати код", + "scanAQrCode": "Сканувати QR-код", + "enterDetailsManually": "Введіть дані вручну", + "edit": "Редагувати", + "copiedToClipboard": "Скопійовано до буфера обміну", + "copiedNextToClipboard": "Наступний код скопійовано до буфера обміну", + "error": "Помилка", + "recoveryKeyCopiedToClipboard": "Ключ відновлення скопійований в буфер обміну", + "recoveryKeyOnForgotPassword": "Якщо ви забудете свій пароль, то єдиний спосіб відновити ваші дані – за допомогою цього ключа.", + "recoveryKeySaveDescription": "Ми не зберігаємо цей ключ, будь ласка, збережіть цей ключ з 24 слів в надійному місці.", + "doThisLater": "Зробити це пізніше", + "saveKey": "Зберегти ключ", + "save": "Зберегти", + "send": "Надіслати", + "saveOrSendDescription": "Чи хочете Ви зберегти це до свого сховища (тека Downloads за замовчуванням), чи надіслати його в інші додатки?", + "saveOnlyDescription": "Чи хочете Ви зберегти це до свого сховища (тека Downloads за замовчуванням)?", + "back": "Назад", + "createAccount": "Створити обліковий запис", + "passwordStrength": "Сила пароля: {passwordStrengthValue}", + "@passwordStrength": { + "description": "Text to indicate the password strength", + "placeholders": { + "passwordStrengthValue": { + "description": "The strength of the password as a string", + "type": "String", + "example": "Weak or Moderate or Strong" + } + }, + "message": "Password Strength: {passwordStrengthText}" + }, + "password": "Пароль", + "signUpTerms": "Я приймаю умови використання і політику конфіденційності", + "privacyPolicyTitle": "Політика конфіденційності", + "termsOfServicesTitle": "Умови", + "encryption": "Шифрування", + "setPasswordTitle": "Встановити пароль", + "changePasswordTitle": "Змінити пароль", + "resetPasswordTitle": "Скинути пароль", + "encryptionKeys": "Ключі шифрування", + "passwordWarning": "Ми не зберігаємо цей пароль, тому, якщо ви його забудете, ми не зможемо розшифрувати Ваші дані", + "enterPasswordToEncrypt": "Введіть пароль, який ми зможемо використати для шифрування ваших даних", + "enterNewPasswordToEncrypt": "Введіть новий пароль, який ми зможемо використати для шифрування ваших даних", + "passwordChangedSuccessfully": "Пароль успішно змінено", + "generatingEncryptionKeys": "Створення ключів шифрування...", + "continueLabel": "Продовжити", + "insecureDevice": "Незахищений пристрій", + "sorryWeCouldNotGenerateSecureKeysOnThisDevicennplease": "На жаль, нам не вдалося згенерувати захищені ключі на цьому пристрої.\n\nБудь ласка, увійдіть з іншого пристрою.", + "howItWorks": "Як це працює", + "ackPasswordLostWarning": "Я розумію, що якщо я втрачу свій пароль, я можу втратити свої дані, тому що вони є захищені наскрізним шифруванням.", + "loginTerms": "Натискаючи «Увійти», я приймаю умови використання і політику конфіденційності", + "logInLabel": "Увійти", + "logout": "Вийти", + "areYouSureYouWantToLogout": "Ви впевнені, що хочете вийти з системи?", + "yesLogout": "Так, вийти з системи", + "exit": "Вийти", + "verifyingRecoveryKey": "Перевірка ключа відновлення...", + "recoveryKeyVerified": "Ключ відновлення перевірено", + "recoveryKeySuccessBody": "Чудово! Ваш ключ відновлення дійсний. Дякуємо за перевірку.\n\nБудь ласка, не забувайте зберігати надійну резервну копію ключа відновлення.", + "invalidRecoveryKey": "Уведений вами ключ відновлення недійсний. Переконайтеся, що він містить 24 слова, а також перевірте правопис кожного.\n\nЯкщо ви ввели старий код відновлення, переконайтеся, що він має 64 символи, а також перевірте кожен з них.", + "recreatePasswordTitle": "Повторно створити пароль", + "recreatePasswordBody": "Поточний пристрій не є достатньо потужним для підтвердження пароля, але ми можемо відновити його таким чином, щоб він працював на всіх пристроях.\n\nБудь ласка, увійдіть за допомогою вашого ключа відновлення і відновіть ваш пароль (ви можете знову використати той самий пароль, якщо бажаєте).", + "invalidKey": "Хибний ключ", + "tryAgain": "Спробуйте ще раз", + "viewRecoveryKey": "Переглянути ключ відновлення", + "confirmRecoveryKey": "Підтвердити ключ відновлення", + "recoveryKeyVerifyReason": "Ваш ключ відновлення це єдиний спосіб відновити ваші фотографії, якщо ви забудете пароль. Ви можете знайти ключ відновлення у меню Налаштування > Обліковий запис.\n\nБудь ласка, введіть ваш ключ відновлення, щоб переконатися, що ви зберегли його правильно.", + "confirmYourRecoveryKey": "Підтвердіть ваш ключ відновлення", + "confirm": "Підтвердити", + "emailYourLogs": "Відправте ваші журнали електронною поштою", + "pleaseSendTheLogsTo": "Будь ласка, надішліть журнали до електронної пошти {toEmail}", + "copyEmailAddress": "Копіювати електронну адресу", + "exportLogs": "Експортувати журнал", + "enterYourRecoveryKey": "Введіть ваш ключ відновлення", + "tempErrorContactSupportIfPersists": "Схоже, що щось пішло не так. Будь ласка, спробуйте ще раз через деякий час. Якщо помилка не зникне, зв'яжіться з нашою командою підтримки.", + "networkHostLookUpErr": "Не вдалося приєднатися до Ente. Будь ласка, перевірте налаштування мережі. Зверніться до нашої команди підтримки, якщо помилка залишиться.", + "networkConnectionRefusedErr": "Не вдалося приєднатися до Ente. Будь ласка, спробуйте ще раз через деякий час. Якщо помилка не зникне, зв'яжіться з нашою командою підтримки.", + "itLooksLikeSomethingWentWrongPleaseRetryAfterSome": "Схоже, що щось пішло не так. Будь ласка, спробуйте ще раз через деякий час. Якщо помилка не зникне, зв'яжіться з нашою командою підтримки.", + "about": "Про додаток", + "weAreOpenSource": "Наш додаток має відкритий код!", + "privacy": "Конфіденційність", + "terms": "Умови", + "checkForUpdates": "Перевiрити наявнiсть оновлень", + "checkStatus": "Перевірити стан", + "downloadUpdate": "Завантажити", + "criticalUpdateAvailable": "Доступне критичне оновлення", + "updateAvailable": "Доступне оновлення", + "update": "Оновлення", + "checking": "Перевірка...", + "youAreOnTheLatestVersion": "Ви використовуєте останню версію", + "warning": "Увага!", + "exportWarningDesc": "Експортований файл містить конфіденційну інформацію. Будь ласка, збережіть його безпечно.", + "iUnderStand": "Я розумію", + "@iUnderStand": { + "description": "Text for the button to confirm the user understands the warning" + }, + "authToExportCodes": "Будь ласка, авторизуйтесь, щоб експортувати ваші коди", + "importSuccessTitle": "Гей, любо!", + "importSuccessDesc": "Ви імпортували наступну кількість кодів: {count}!", + "@importSuccessDesc": { + "placeholders": { + "count": { + "description": "The number of codes imported", + "type": "int", + "example": "1" + } + } + }, + "sorry": "Дуже шкода", + "importFailureDesc": "Не вдалося обробити обраний файл.\nБудь ласка, надішліть електронну пошту на адресу support@ente.io, якщо вам потрібна допомога!", + "pendingSyncs": "Увага!", + "pendingSyncsWarningBody": "Деякі з ваших кодів не були збережені.\n\nБудь ласка, переконайтеся, що у вас є резервна копія для цих кодів перед виходом.", + "checkInboxAndSpamFolder": "Будь ласка, перевірте вашу скриньку електронної пошти (та спам), щоб завершити перевірку", + "tapToEnterCode": "Натисніть, щоб ввести код", + "resendEmail": "Повторно надіслати лист на електронну пошту", + "weHaveSendEmailTo": "Ми надіслали листа на адресу електронної пошти {email}", + "@weHaveSendEmailTo": { + "description": "Text to indicate that we have sent a mail to the user", + "placeholders": { + "email": { + "description": "The email address of the user", + "type": "String", + "example": "example@ente.io" + } + } + }, + "activeSessions": "Активні сеанси", + "somethingWentWrongPleaseTryAgain": "Щось пішло не так, спробуйте, будь ласка, знову", + "thisWillLogYouOutOfThisDevice": "Це призведе до виходу на цьому пристрої!", + "thisWillLogYouOutOfTheFollowingDevice": "Це призведе до виходу на наступному пристрої:", + "terminateSession": "Припинити сеанс?", + "terminate": "Припинити", + "thisDevice": "Цей пристрій", + "toResetVerifyEmail": "Щоб скинути пароль, будь ласка, спочатку підтвердіть адресу своєї електронної пошти.", + "thisEmailIsAlreadyInUse": "Ця адреса електронної пошти вже використовується", + "verificationFailedPleaseTryAgain": "Перевірка не вдалася, спробуйте ще", + "yourVerificationCodeHasExpired": "Час дії коду підтвердження минув", + "incorrectCode": "Невірний код", + "sorryTheCodeYouveEnteredIsIncorrect": "Вибачте, але введений вами код є невірним", + "emailChangedTo": "Адресу електронної пошти змінено на {newEmail}", + "authenticationFailedPleaseTryAgain": "Аутентифікація не пройдена. Будь ласка, спробуйте ще раз", + "authenticationSuccessful": "Автентифікацію виконано!", + "twofactorAuthenticationSuccessfullyReset": "Двофакторна аутентифікація успішно скинута", + "incorrectRecoveryKey": "Неправильний ключ відновлення", + "theRecoveryKeyYouEnteredIsIncorrect": "Ви ввели неправильний ключ відновлення", + "enterPassword": "Введіть пароль", + "selectExportFormat": "Виберіть формат експортування", + "exportDialogDesc": "Зашифрований експорт буде захищений паролем, який Ви виберете.", + "encrypted": "Зашифрований", + "plainText": "Звичайний текст", + "passwordToEncryptExport": "Пароль для зашифровування експорту", + "export": "Експорт", + "useOffline": "Використовувати без резервних копій", + "signInToBackup": "Увійдіть для резервного копіювання кодів", + "singIn": "Увійти", + "sigInBackupReminder": "Будь ласка, експортуйте свої коди, щоб зберегти резервну копію, з якої ви зможете їх відновити.", + "offlineModeWarning": "Ви збираєтеся продовжити без резервних копій. Будь ласка, зробіть ручну резервну копію, щоб переконатися, що ваші коди в безпеці.", + "showLargeIcons": "Показувати великі іконки", + "shouldHideCode": "Приховати коди", + "doubleTapToViewHiddenCode": "Ви можете двічі натиснути на запис для перегляду коду", + "focusOnSearchBar": "Сфокусуватися на пошуку після запуску програми", + "confirmUpdatingkey": "Ви впевнені у тому, що бажаєте змінити секретний ключ?", + "minimizeAppOnCopy": "Згорнути програму після копіювання", + "editCodeAuthMessage": "Аутентифікуйтесь, щоб змінити код", + "deleteCodeAuthMessage": "Аутентифікуйтесь, щоб видалити код", + "showQRAuthMessage": "Аутентифікуйтесь, щоб показати QR-код", + "confirmAccountDeleteTitle": "Підтвердіть видалення облікового запису", + "confirmAccountDeleteMessage": "Цей обліковий запис є зв'язаним з іншими програмами Ente, якщо ви використовуєте якісь з них.\n\nВаші завантажені дані у всіх програмах Ente будуть заплановані до видалення, а обліковий запис буде видалено назавжди.", + "androidBiometricHint": "Підтвердити ідентифікацію", + "@androidBiometricHint": { + "description": "Hint message advising the user how to authenticate with biometrics. It is used on Android side. Maximum 60 characters." + }, + "androidBiometricNotRecognized": "Не розпізнано. Спробуйте ще раз.", + "@androidBiometricNotRecognized": { + "description": "Message to let the user know that authentication was failed. It is used on Android side. Maximum 60 characters." + }, + "androidBiometricSuccess": "Успіх", + "@androidBiometricSuccess": { + "description": "Message to let the user know that authentication was successful. It is used on Android side. Maximum 60 characters." + }, + "androidCancelButton": "Скасувати", + "@androidCancelButton": { + "description": "Message showed on a button that the user can click to leave the current dialog. It is used on Android side. Maximum 30 characters." + }, + "androidSignInTitle": "Необхідна аутентифікація", + "@androidSignInTitle": { + "description": "Message showed as a title in a dialog which indicates the user that they need to scan biometric to continue. It is used on Android side. Maximum 60 characters." + }, + "androidBiometricRequiredTitle": "Потрібна біометрична аутентифікація", + "@androidBiometricRequiredTitle": { + "description": "Message showed as a title in a dialog which indicates the user has not set up biometric authentication on their device. It is used on Android side. Maximum 60 characters." + }, + "androidDeviceCredentialsRequiredTitle": "Необхідні облікові дані пристрою", + "@androidDeviceCredentialsRequiredTitle": { + "description": "Message showed as a title in a dialog which indicates the user has not set up credentials authentication on their device. It is used on Android side. Maximum 60 characters." + }, + "androidDeviceCredentialsSetupDescription": "Необхідні облікові дані пристрою", + "@androidDeviceCredentialsSetupDescription": { + "description": "Message advising the user to go to the settings and configure device credentials on their device. It shows in a dialog on Android side." + }, + "goToSettings": "Перейти до налаштувань", + "@goToSettings": { + "description": "Message showed on a button that the user can click to go to settings pages from the current dialog. It is used on both Android and iOS side. Maximum 30 characters." + }, + "androidGoToSettingsDescription": "Біометрична аутентифікація не налаштована на вашому пристрої. Перейдіть в 'Налаштування > Безпека', щоб додати біометричну аутентифікацію.", + "@androidGoToSettingsDescription": { + "description": "Message advising the user to go to the settings and configure biometric on their device. It shows in a dialog on Android side." + }, + "iOSLockOut": "Біометрична автентифікація вимкнена. Будь ласка, заблокуйте і розблокуйте свій екран, щоб увімкнути її.", + "@iOSLockOut": { + "description": "Message advising the user to re-enable biometrics on their device. It shows in a dialog on iOS side." + }, + "iOSGoToSettingsDescription": "Біометрична аутентифікація не налаштована на вашому пристрої. Увімкніть TouchID або FaceID на вашому телефоні.", + "@iOSGoToSettingsDescription": { + "description": "Message advising the user to go to the settings and configure Biometrics for their device. It shows in a dialog on iOS side." + }, + "iOSOkButton": "OK", + "@iOSOkButton": { + "description": "Message showed on a button that the user can click to leave the current dialog. It is used on iOS side. Maximum 30 characters." + }, + "noInternetConnection": "Немає підключення до Інтернету", + "pleaseCheckYourInternetConnectionAndTryAgain": "Будь ласка, перевірте підключення до Інтернету та спробуйте ще раз.", + "signOutFromOtherDevices": "Вийти на інших пристроях", + "signOutOtherBody": "Якщо ви думаєте, що хтось може знати ваш пароль, ви можете примусити всі інші пристрої, які використовують ваш обліковий запис, вийти з нього.", + "signOutOtherDevices": "Вийти на інших пристроях", + "doNotSignOut": "Не виходити", + "hearUsWhereTitle": "Як ви дізналися про Ente? (опціонально)", + "hearUsExplanation": "Ми не відстежуємо встановлення додатків. Але, якщо ви скажете нам, де ви нас знайшли, це допоможе!", + "recoveryKeySaved": "Ключ відновлення збережений у теці Downloads!", + "waitingForBrowserRequest": "Очікування запиту браузера...", + "waitingForVerification": "Очікується підтвердження...", + "passkey": "Ключ доступу", + "passKeyPendingVerification": "Підтвердження все ще в процесі", + "loginSessionExpired": "Час сеансу минув", + "loginSessionExpiredDetails": "Термін дії вашого сеансу завершився. Будь ласка, увійдіть знову.", + "developerSettingsWarning": "Ви впевнені, що хочете змінити налаштування розробника?", + "developerSettings": "Налаштування розробника", + "serverEndpoint": "Кінцева точка сервера", + "invalidEndpoint": "Некоректна кінцева точка", + "invalidEndpointMessage": "Вибачте, введена вами кінцева точка є недійсною. Введіть дійсну кінцеву точку та спробуйте ще раз.", + "endpointUpdatedMessage": "Точка входу успішно оновлена", + "customEndpoint": "Приєднано до {endpoint}", + "pinText": "Закріпити", + "unpinText": "Відкріпити", + "pinnedCodeMessage": "{code} закріплено", + "unpinnedCodeMessage": "{code} відкріплено", + "tags": "Мітки", + "createNewTag": "Створити нову мітку", + "tag": "Мітка", + "create": "Створити", + "editTag": "Редагувати мітку", + "deleteTagTitle": "Видалити мітку?", + "deleteTagMessage": "Ви впевнені, що хочете видалити цю мітку? Ця дія є незворотною.", + "somethingWentWrongParsingCode": "Не вдалося обробити цю кількість кодів: {x}.", + "updateNotAvailable": "Оновлення недоступне", + "viewRawCodes": "Переглянути коди як є", + "rawCodes": "Коди як є", + "rawCodeData": "Дані кодів як є", + "appLock": "Блокування", + "noSystemLockFound": "Не знайдено системного блокування", + "toEnableAppLockPleaseSetupDevicePasscodeOrScreen": "Для увімкнення блокування програми, будь ласка, налаштуйте пароль пристрою або блокування екрана в системних налаштуваннях.", + "autoLock": "Автоблокування", + "immediately": "Негайно", + "reEnterPassword": "Введіть пароль ще раз", + "reEnterPin": "Введіть PIN-код ще раз", + "next": "Наступний", + "tooManyIncorrectAttempts": "Завелика кількість невірних спроб", + "tapToUnlock": "Доторкніться, щоб розблокувати", + "setNewPassword": "Встановити новий пароль", + "deviceLock": "Блокування пристрою", + "hideContent": "Приховати зміст", + "hideContentDescriptionAndroid": "Приховує зміст програми в перемикачі програм і вимикає скриншоти", + "hideContentDescriptioniOS": "Приховує зміст в перемикачі додатків", + "autoLockFeatureDescription": "Час, через який додаток буде заблоковано після розміщення у фоновому режимі", + "appLockDescription": "Виберіть між типовим екраном блокування вашого пристрою та власним екраном блокування з PIN-кодом або паролем.", + "pinLock": "PIN-код", + "enterPin": "Введіть PIN-код", + "setNewPin": "Встановити новий PIN-код", + "importFailureDescNew": "Не вдалося обробити вибраний файл." +} \ No newline at end of file diff --git a/auth/lib/l10n/arb/app_zh.arb b/auth/lib/l10n/arb/app_zh.arb index ac8d9e5d7a..dd093ea166 100644 --- a/auth/lib/l10n/arb/app_zh.arb +++ b/auth/lib/l10n/arb/app_zh.arb @@ -465,5 +465,6 @@ "appLockDescription": "在设备的默认锁定屏幕和带有 PIN 或密码的自定义锁定屏幕之间进行选择。", "pinLock": "Pin 锁定", "enterPin": "输入 PIN 码", - "setNewPin": "设置新 PIN 码" + "setNewPin": "设置新 PIN 码", + "importFailureDescNew": "无法解析选定的文件。" } \ No newline at end of file From 43e711274edd2d90970bb4b1b8898804d73cc564 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Mon, 12 Aug 2024 12:39:41 +0530 Subject: [PATCH 178/211] Add an icon to open the active location on Maps --- mobile/ios/Podfile.lock | 30 ++++++++------ mobile/ios/Runner.xcodeproj/project.pbxproj | 10 +++-- mobile/lib/ui/map/map_view.dart | 19 +++++++++ mobile/pubspec.lock | 44 ++++++++++++--------- mobile/pubspec.yaml | 1 + 5 files changed, 70 insertions(+), 34 deletions(-) diff --git a/mobile/ios/Podfile.lock b/mobile/ios/Podfile.lock index 3fee5e8c6e..ad83ed9c70 100644 --- a/mobile/ios/Podfile.lock +++ b/mobile/ios/Podfile.lock @@ -3,13 +3,13 @@ PODS: - Flutter - battery_info (0.0.1): - Flutter + - blurhash_ffi (0.0.1): + - Flutter - connectivity_plus (0.0.1): - Flutter - FlutterMacOS - dart_ui_isolate (0.0.1): - Flutter - - dchs_motion_sensors (0.0.1): - - Flutter - device_info_plus (0.0.1): - Flutter - ffmpeg-kit-ios-min (6.0) @@ -148,6 +148,8 @@ PODS: - Flutter - media_kit_video (0.0.1): - Flutter + - motion_sensors (0.0.1): + - Flutter - motionphoto (0.0.1): - Flutter - move_to_background (0.0.1): @@ -178,6 +180,8 @@ PODS: - photo_manager (2.0.0): - Flutter - FlutterMacOS + - privacy_screen (0.0.1): + - Flutter - PromisesObjC (2.4.0) - receive_sharing_intent (1.6.8): - Flutter @@ -201,8 +205,6 @@ PODS: - shared_preferences_foundation (0.0.1): - Flutter - FlutterMacOS - - smart_auth (0.0.1): - - Flutter - sqflite (0.0.3): - Flutter - FlutterMacOS @@ -240,9 +242,9 @@ PODS: DEPENDENCIES: - background_fetch (from `.symlinks/plugins/background_fetch/ios`) - battery_info (from `.symlinks/plugins/battery_info/ios`) + - blurhash_ffi (from `.symlinks/plugins/blurhash_ffi/ios`) - connectivity_plus (from `.symlinks/plugins/connectivity_plus/darwin`) - dart_ui_isolate (from `.symlinks/plugins/dart_ui_isolate/ios`) - - dchs_motion_sensors (from `.symlinks/plugins/dchs_motion_sensors/ios`) - device_info_plus (from `.symlinks/plugins/device_info_plus/ios`) - ffmpeg_kit_flutter_min (from `.symlinks/plugins/ffmpeg_kit_flutter_min/ios`) - file_saver (from `.symlinks/plugins/file_saver/ios`) @@ -269,6 +271,7 @@ DEPENDENCIES: - media_kit_libs_ios_video (from `.symlinks/plugins/media_kit_libs_ios_video/ios`) - media_kit_native_event_loop (from `.symlinks/plugins/media_kit_native_event_loop/ios`) - media_kit_video (from `.symlinks/plugins/media_kit_video/ios`) + - motion_sensors (from `.symlinks/plugins/motion_sensors/ios`) - motionphoto (from `.symlinks/plugins/motionphoto/ios`) - move_to_background (from `.symlinks/plugins/move_to_background/ios`) - onnxruntime (from `.symlinks/plugins/onnxruntime/ios`) @@ -277,12 +280,12 @@ DEPENDENCIES: - path_provider_foundation (from `.symlinks/plugins/path_provider_foundation/darwin`) - permission_handler_apple (from `.symlinks/plugins/permission_handler_apple/ios`) - photo_manager (from `.symlinks/plugins/photo_manager/ios`) + - privacy_screen (from `.symlinks/plugins/privacy_screen/ios`) - receive_sharing_intent (from `.symlinks/plugins/receive_sharing_intent/ios`) - screen_brightness_ios (from `.symlinks/plugins/screen_brightness_ios/ios`) - sentry_flutter (from `.symlinks/plugins/sentry_flutter/ios`) - share_plus (from `.symlinks/plugins/share_plus/ios`) - shared_preferences_foundation (from `.symlinks/plugins/shared_preferences_foundation/darwin`) - - smart_auth (from `.symlinks/plugins/smart_auth/ios`) - sqflite (from `.symlinks/plugins/sqflite/darwin`) - sqlite3_flutter_libs (from `.symlinks/plugins/sqlite3_flutter_libs/ios`) - uni_links (from `.symlinks/plugins/uni_links/ios`) @@ -321,12 +324,12 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/background_fetch/ios" battery_info: :path: ".symlinks/plugins/battery_info/ios" + blurhash_ffi: + :path: ".symlinks/plugins/blurhash_ffi/ios" connectivity_plus: :path: ".symlinks/plugins/connectivity_plus/darwin" dart_ui_isolate: :path: ".symlinks/plugins/dart_ui_isolate/ios" - dchs_motion_sensors: - :path: ".symlinks/plugins/dchs_motion_sensors/ios" device_info_plus: :path: ".symlinks/plugins/device_info_plus/ios" ffmpeg_kit_flutter_min: @@ -379,6 +382,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/media_kit_native_event_loop/ios" media_kit_video: :path: ".symlinks/plugins/media_kit_video/ios" + motion_sensors: + :path: ".symlinks/plugins/motion_sensors/ios" motionphoto: :path: ".symlinks/plugins/motionphoto/ios" move_to_background: @@ -395,6 +400,8 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/permission_handler_apple/ios" photo_manager: :path: ".symlinks/plugins/photo_manager/ios" + privacy_screen: + :path: ".symlinks/plugins/privacy_screen/ios" receive_sharing_intent: :path: ".symlinks/plugins/receive_sharing_intent/ios" screen_brightness_ios: @@ -405,8 +412,6 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/share_plus/ios" shared_preferences_foundation: :path: ".symlinks/plugins/shared_preferences_foundation/darwin" - smart_auth: - :path: ".symlinks/plugins/smart_auth/ios" sqflite: :path: ".symlinks/plugins/sqflite/darwin" sqlite3_flutter_libs: @@ -427,9 +432,9 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: background_fetch: 2319bf7e18237b4b269430b7f14d177c0df09c5a battery_info: 09f5c9ee65394f2291c8c6227bedff345b8a730c + blurhash_ffi: 4831b96320d4273876c9a2fd3f7d50b8a3a53509 connectivity_plus: ddd7f30999e1faaef5967c23d5b6d503d10434db dart_ui_isolate: d5bcda83ca4b04f129d70eb90110b7a567aece14 - dchs_motion_sensors: 9cef816635a39345cda9f0c4943e061f6429f453 device_info_plus: c6fb39579d0f423935b0c9ce7ee2f44b71b9fce6 ffmpeg-kit-ios-min: 4e9a088f4ee9629435960b9d68e54848975f1931 ffmpeg_kit_flutter_min: 5eff47f4965bf9d1150e98961eb6129f5ae3f28c @@ -466,6 +471,7 @@ SPEC CHECKSUMS: media_kit_libs_ios_video: a5fe24bc7875ccd6378a0978c13185e1344651c1 media_kit_native_event_loop: e6b2ab20cf0746eb1c33be961fcf79667304fa2a media_kit_video: 5da63f157170e5bf303bf85453b7ef6971218a2e + motion_sensors: 03f55b7c637a7e365a0b5f9697a449f9059d5d91 motionphoto: d4a432b8c8f22fb3ad966258597c0103c9c5ff16 move_to_background: 39a5b79b26d577b0372cbe8a8c55e7aa9fcd3a2d nanopb: 438bc412db1928dac798aa6fd75726007be04262 @@ -478,6 +484,7 @@ SPEC CHECKSUMS: path_provider_foundation: 3784922295ac71e43754bd15e0653ccfd36a147c permission_handler_apple: 9878588469a2b0d0fc1e048d9f43605f92e6cec2 photo_manager: ff695c7a1dd5bc379974953a2b5c0a293f7c4c8a + privacy_screen: 1a131c052ceb3c3659934b003b0d397c2381a24e PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47 receive_sharing_intent: 6837b01768e567fe8562182397bf43d63d8c6437 screen_brightness_ios: 715ca807df953bf676d339f11464e438143ee625 @@ -488,7 +495,6 @@ SPEC CHECKSUMS: SentryPrivate: d651efb234cf385ec9a1cdd3eff94b5e78a0e0fe share_plus: 8875f4f2500512ea181eef553c3e27dba5135aad shared_preferences_foundation: b4c3b4cddf1c21f02770737f147a3f5da9d39695 - smart_auth: 4bedbc118723912d0e45a07e8ab34039c19e04f2 sqflite: 673a0e54cc04b7d6dba8d24fb8095b31c3a99eec sqlite3: 02d1f07eaaa01f80a1c16b4b31dfcbb3345ee01a sqlite3_flutter_libs: af0e8fe9bce48abddd1ffdbbf839db0302d72d80 diff --git a/mobile/ios/Runner.xcodeproj/project.pbxproj b/mobile/ios/Runner.xcodeproj/project.pbxproj index 2c954cdfce..1414f8d947 100644 --- a/mobile/ios/Runner.xcodeproj/project.pbxproj +++ b/mobile/ios/Runner.xcodeproj/project.pbxproj @@ -292,9 +292,9 @@ "${BUILT_PRODUCTS_DIR}/Toast/Toast.framework", "${BUILT_PRODUCTS_DIR}/background_fetch/background_fetch.framework", "${BUILT_PRODUCTS_DIR}/battery_info/battery_info.framework", + "${BUILT_PRODUCTS_DIR}/blurhash_ffi/blurhash_ffi.framework", "${BUILT_PRODUCTS_DIR}/connectivity_plus/connectivity_plus.framework", "${BUILT_PRODUCTS_DIR}/dart_ui_isolate/dart_ui_isolate.framework", - "${BUILT_PRODUCTS_DIR}/dchs_motion_sensors/dchs_motion_sensors.framework", "${BUILT_PRODUCTS_DIR}/device_info_plus/device_info_plus.framework", "${BUILT_PRODUCTS_DIR}/file_saver/file_saver.framework", "${BUILT_PRODUCTS_DIR}/fk_user_agent/fk_user_agent.framework", @@ -318,6 +318,7 @@ "${BUILT_PRODUCTS_DIR}/media_kit_libs_ios_video/media_kit_libs_ios_video.framework", "${BUILT_PRODUCTS_DIR}/media_kit_native_event_loop/media_kit_native_event_loop.framework", "${BUILT_PRODUCTS_DIR}/media_kit_video/media_kit_video.framework", + "${BUILT_PRODUCTS_DIR}/motion_sensors/motion_sensors.framework", "${BUILT_PRODUCTS_DIR}/motionphoto/motionphoto.framework", "${BUILT_PRODUCTS_DIR}/move_to_background/move_to_background.framework", "${BUILT_PRODUCTS_DIR}/nanopb/nanopb.framework", @@ -325,12 +326,12 @@ "${BUILT_PRODUCTS_DIR}/package_info_plus/package_info_plus.framework", "${BUILT_PRODUCTS_DIR}/path_provider_foundation/path_provider_foundation.framework", "${BUILT_PRODUCTS_DIR}/photo_manager/photo_manager.framework", + "${BUILT_PRODUCTS_DIR}/privacy_screen/privacy_screen.framework", "${BUILT_PRODUCTS_DIR}/receive_sharing_intent/receive_sharing_intent.framework", "${BUILT_PRODUCTS_DIR}/screen_brightness_ios/screen_brightness_ios.framework", "${BUILT_PRODUCTS_DIR}/sentry_flutter/sentry_flutter.framework", "${BUILT_PRODUCTS_DIR}/share_plus/share_plus.framework", "${BUILT_PRODUCTS_DIR}/shared_preferences_foundation/shared_preferences_foundation.framework", - "${BUILT_PRODUCTS_DIR}/smart_auth/smart_auth.framework", "${BUILT_PRODUCTS_DIR}/sqflite/sqflite.framework", "${BUILT_PRODUCTS_DIR}/sqlite3/sqlite3.framework", "${BUILT_PRODUCTS_DIR}/sqlite3_flutter_libs/sqlite3_flutter_libs.framework", @@ -385,9 +386,9 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/Toast.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/background_fetch.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/battery_info.framework", + "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/blurhash_ffi.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/connectivity_plus.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/dart_ui_isolate.framework", - "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/dchs_motion_sensors.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/device_info_plus.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/file_saver.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/fk_user_agent.framework", @@ -411,6 +412,7 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_kit_libs_ios_video.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_kit_native_event_loop.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/media_kit_video.framework", + "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/motion_sensors.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/motionphoto.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/move_to_background.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/nanopb.framework", @@ -418,12 +420,12 @@ "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/package_info_plus.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/path_provider_foundation.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/photo_manager.framework", + "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/privacy_screen.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/receive_sharing_intent.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/screen_brightness_ios.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sentry_flutter.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/share_plus.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/shared_preferences_foundation.framework", - "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/smart_auth.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sqflite.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sqlite3.framework", "${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/sqlite3_flutter_libs.framework", diff --git a/mobile/lib/ui/map/map_view.dart b/mobile/lib/ui/map/map_view.dart index d4de022197..8804774223 100644 --- a/mobile/lib/ui/map/map_view.dart +++ b/mobile/lib/ui/map/map_view.dart @@ -2,6 +2,7 @@ import "package:flutter/material.dart"; import "package:flutter_map/flutter_map.dart"; import "package:flutter_map_marker_cluster/flutter_map_marker_cluster.dart"; import "package:latlong2/latlong.dart"; +import "package:maps_launcher/maps_launcher.dart"; import "package:photos/ui/map/image_marker.dart"; import "package:photos/ui/map/map_button.dart"; import "package:photos/ui/map/map_gallery_tile.dart"; @@ -157,6 +158,24 @@ class _MapViewState extends State { ), ) : const SizedBox.shrink(), + widget.showControls + ? Positioned( + top: 4, + right: 10, + child: SafeArea( + child: MapButton( + icon: Icons.navigation_outlined, + onPressed: () { + MapsLauncher.launchCoordinates( + widget.controller.camera.center.latitude, + widget.controller.camera.center.longitude, + ); + }, + heroTag: 'open-map', + ), + ), + ) + : const SizedBox.shrink(), widget.showControls ? Positioned( bottom: widget.bottomSheetDraggableAreaHeight + 10, diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 5adbd5c31c..6c064b81cb 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -1304,18 +1304,18 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "3f87a60e8c63aecc975dda1ceedbc8f24de75f09e4856ea27daf8958f2f0ce05" + sha256: "7f0df31977cb2c0b88585095d168e689669a2cc9b97c309665e3386f3e9d341a" url: "https://pub.dev" source: hosted - version: "10.0.5" + version: "10.0.4" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: "932549fb305594d82d7183ecd9fa93463e9914e1b67cacc34bc40906594a1806" + sha256: "06e98f569d004c1315b991ded39924b21af84cf14cc94791b8aea337d25b57f8" url: "https://pub.dev" source: hosted - version: "3.0.5" + version: "3.0.3" leak_tracker_testing: dependency: transitive description: @@ -1428,6 +1428,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.4.3" + maps_launcher: + dependency: "direct main" + description: + name: maps_launcher + sha256: "57ba3c31db96e30f58c23fcb22a1fac6accc5683535b2cf344c534bbb9f8f910" + url: "https://pub.dev" + source: hosted + version: "2.2.1" matcher: dependency: transitive description: @@ -1440,10 +1448,10 @@ packages: dependency: transitive description: name: material_color_utilities - sha256: f7142bb1154231d7ea5f96bc7bde4bda2a0945d2806bb11670e30b850d56bdec + sha256: "0e0a020085b65b6083975e499759762399b4475f766c21668c4ecca34ea74e5a" url: "https://pub.dev" source: hosted - version: "0.11.1" + version: "0.8.0" media_extension: dependency: "direct main" description: @@ -1528,10 +1536,10 @@ packages: dependency: transitive description: name: meta - sha256: bdb68674043280c3428e9ec998512fb681678676b3c54e773629ffe74419f8c7 + sha256: "7687075e408b093f36e6bbf6c91878cc0d4cd10f409506f7bc996f68220b9136" url: "https://pub.dev" source: hosted - version: "1.15.0" + version: "1.12.0" mgrs_dart: dependency: transitive description: @@ -1877,10 +1885,10 @@ packages: dependency: transitive description: name: platform - sha256: "9b71283fc13df574056616011fb138fd3b793ea47cc509c189a6c3fa5f8a1a65" + sha256: "12220bb4b65720483f8fa9450b4332347737cf8213dd2840d8b2c823e47243ec" url: "https://pub.dev" source: hosted - version: "3.1.5" + version: "3.1.4" plugin_platform_interface: dependency: transitive description: @@ -2386,26 +2394,26 @@ packages: dependency: "direct dev" description: name: test - sha256: "7ee44229615f8f642b68120165ae4c2a75fe77ae2065b1e55ae4711f6cf0899e" + sha256: "7ee446762c2c50b3bd4ea96fe13ffac69919352bd3b4b17bac3f3465edc58073" url: "https://pub.dev" source: hosted - version: "1.25.7" + version: "1.25.2" test_api: dependency: transitive description: name: test_api - sha256: "5b8a98dafc4d5c4c9c72d8b31ab2b23fc13422348d2997120294d3bac86b4ddb" + sha256: "9955ae474176f7ac8ee4e989dadfb411a58c30415bcfb648fa04b2b8a03afa7f" url: "https://pub.dev" source: hosted - version: "0.7.2" + version: "0.7.0" test_core: dependency: transitive description: name: test_core - sha256: "55ea5a652e38a1dfb32943a7973f3681a60f872f8c3a05a14664ad54ef9c6696" + sha256: "2bc4b4ecddd75309300d8096f781c0e3280ca1ef85beda558d33fcbedc2eead4" url: "https://pub.dev" source: hosted - version: "0.6.4" + version: "0.6.0" timezone: dependency: transitive description: @@ -2684,10 +2692,10 @@ packages: dependency: transitive description: name: vm_service - sha256: f652077d0bdf60abe4c1f6377448e8655008eef28f128bc023f7b5e8dfeb48fc + sha256: "3923c89304b715fb1eb6423f017651664a03bf5f4b29983627c4da791f74a4ec" url: "https://pub.dev" source: hosted - version: "14.2.4" + version: "14.2.1" volume_controller: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index d3a122d259..70149902c8 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -113,6 +113,7 @@ dependencies: local_auth_ios: logging: ^1.0.1 lottie: ^1.2.2 + maps_launcher: ^2.2.1 media_extension: ^1.0.1 media_kit: ^1.1.10+1 media_kit_libs_video: ^1.0.4 From ec8bd5bc7f7f3f7d97bf244a15bdb8f243199819 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Mon, 12 Aug 2024 12:46:25 +0530 Subject: [PATCH 179/211] Fix lint warnings --- mobile/lib/utils/email_util.dart | 2 +- mobile/lib/utils/file_uploader_util.dart | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mobile/lib/utils/email_util.dart b/mobile/lib/utils/email_util.dart index 7d085cbf69..c500516ca0 100644 --- a/mobile/lib/utils/email_util.dart +++ b/mobile/lib/utils/email_util.dart @@ -153,7 +153,7 @@ Future getZippedLogsFile(BuildContext? context) async { final encoder = ZipFileEncoder(); encoder.create(zipFilePath); await encoder.addDirectory(logsDirectory); - encoder.close(); + await encoder.close(); if (context != null) { await dialog.hide(); } diff --git a/mobile/lib/utils/file_uploader_util.dart b/mobile/lib/utils/file_uploader_util.dart index e443b1ad8c..6106307fd2 100644 --- a/mobile/lib/utils/file_uploader_util.dart +++ b/mobile/lib/utils/file_uploader_util.dart @@ -185,7 +185,7 @@ Future _computeZip(Map args) async { encoder.create(zipPath); await encoder.addFile(File(imagePath), "image" + extension(imagePath)); await encoder.addFile(File(videoPath), "video" + extension(videoPath)); - encoder.close(); + await encoder.close(); } Future zip({ From bcf29d971e884f1647d7df1de462262956f015d7 Mon Sep 17 00:00:00 2001 From: vishnukvmd Date: Mon, 12 Aug 2024 12:49:45 +0530 Subject: [PATCH 180/211] Fix more warnings --- mobile/lib/ui/payment/store_subscription_page.dart | 2 +- mobile/lib/ui/payment/stripe_subscription_page.dart | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/mobile/lib/ui/payment/store_subscription_page.dart b/mobile/lib/ui/payment/store_subscription_page.dart index fc7fa5adeb..86d29fd715 100644 --- a/mobile/lib/ui/payment/store_subscription_page.dart +++ b/mobile/lib/ui/payment/store_subscription_page.dart @@ -309,7 +309,7 @@ class _StoreSubscriptionPageState extends State { Padding( padding: const EdgeInsets.fromLTRB(16, 40, 16, 4), child: MenuItemWidget( - captionedTextWidget: CaptionedTextWidget( + captionedTextWidget: const CaptionedTextWidget( title: "Manage payment method", ), menuItemColor: colorScheme.fillFaint, diff --git a/mobile/lib/ui/payment/stripe_subscription_page.dart b/mobile/lib/ui/payment/stripe_subscription_page.dart index 7ab1a0f9cc..93479b8038 100644 --- a/mobile/lib/ui/payment/stripe_subscription_page.dart +++ b/mobile/lib/ui/payment/stripe_subscription_page.dart @@ -55,7 +55,6 @@ class _StripeSubscriptionPageState extends State { // indicates if user's subscription plan is still active late bool _hasActiveSubscription; bool _hideCurrentPlanSelection = false; - late FreePlan _freePlan; List _plans = []; bool _hasLoadedData = false; bool _isLoading = false; @@ -92,7 +91,6 @@ class _StripeSubscriptionPageState extends State { // _filterPlansForUI is used for initializing initState & plan toggle states Future _filterStripeForUI() async { final billingPlans = await _billingService.getBillingPlans(); - _freePlan = billingPlans.freePlan; _plans = billingPlans.plans.where((plan) { if (plan.stripeID.isEmpty) { return false; @@ -279,7 +277,7 @@ class _StripeSubscriptionPageState extends State { Padding( padding: const EdgeInsets.fromLTRB(16, 2, 16, 2), child: MenuItemWidget( - captionedTextWidget: CaptionedTextWidget( + captionedTextWidget: const CaptionedTextWidget( title: "Manage payment method", ), menuItemColor: colorScheme.fillFaint, From bbac3a2a94e09efe12a4380ff8079e0d5b3bea38 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 12:54:58 +0530 Subject: [PATCH 181/211] Update key --- web/packages/new/photos/services/ml/db.ts | 6 ++++++ web/packages/new/photos/services/ml/index.ts | 20 +++++++++----------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/web/packages/new/photos/services/ml/db.ts b/web/packages/new/photos/services/ml/db.ts index 3fe18d0731..dc65270a08 100644 --- a/web/packages/new/photos/services/ml/db.ts +++ b/web/packages/new/photos/services/ml/db.ts @@ -149,6 +149,12 @@ const deleteLegacyDB = () => { removeKV("embeddingSyncTime:onnx-clip"), removeKV("embeddingSyncTime:file-ml-clip-face"), ]); + + // Delete legacy ML keys. + // + // This code was added August 2024 (v1.7.3-beta) and can be removed at some + // point when most clients have migrated (tag: Migration). + localStorage.removeItem("faceIndexingEnabled"); }; /** diff --git a/web/packages/new/photos/services/ml/index.ts b/web/packages/new/photos/services/ml/index.ts index 9355e7d24b..d998029507 100644 --- a/web/packages/new/photos/services/ml/index.ts +++ b/web/packages/new/photos/services/ml/index.ts @@ -160,13 +160,8 @@ export const isMLSupported = isDesktop; /** * TODO-ML: This will not be needed when we move to a public beta. - * Was this someone who might've enabled the beta ML? If so, show them the - * coming back soon banner while we finalize it. */ -export const canEnableML = async () => - // TODO-ML: The interim condition should be - // isDevBuild || (await isInternalUser()) || (await isBetaUser()); - await isInternalUser(); +export const canEnableML = async () => await isInternalUser(); /** * Initialize the ML subsystem if the user has enabled it in preferences. @@ -224,6 +219,11 @@ export const disableML = async () => { triggerStatusUpdate(); }; +/** + * Local storage key for {@link isMLEnabledLocal}. + */ +const mlLocalKey = "mlEnabled"; + /** * Return true if our local persistence thinks that ML is enabled. * @@ -233,17 +233,15 @@ export const disableML = async () => { * The remote status is tracked with a separate {@link isMLEnabledRemote} flag * that is synced with remote. */ -const isMLEnabledLocal = () => - // TODO-ML: Rename this flag - localStorage.getItem("faceIndexingEnabled") == "1"; +const isMLEnabledLocal = () => localStorage.getItem(mlLocalKey) == "1"; /** * Update the (locally stored) value of {@link isMLEnabledLocal}. */ const setIsMLEnabledLocal = (enabled: boolean) => enabled - ? localStorage.setItem("faceIndexingEnabled", "1") - : localStorage.removeItem("faceIndexingEnabled"); + ? localStorage.setItem(mlLocalKey, "1") + : localStorage.removeItem(mlLocalKey); /** * For historical reasons, this is called "faceSearchEnabled" (it started off as From 4a13b04b1cbf8154c319ff0c01cf4cd19e57563a Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:03:51 +0530 Subject: [PATCH 182/211] [server] Update db script --- server/migrations/89_file_data_table.up.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/server/migrations/89_file_data_table.up.sql b/server/migrations/89_file_data_table.up.sql index 72bfe0979d..d67d05e6e4 100644 --- a/server/migrations/89_file_data_table.up.sql +++ b/server/migrations/89_file_data_table.up.sql @@ -1,7 +1,7 @@ -ALTER TABLE temp_objects -ADD COLUMN IF NOT EXISTS bucket_id s3region; -ALTER TYPE OBJECT_TYPE ADD VALUE 'derivedMeta'; +ALTER TABLE temp_objects ADD COLUMN IF NOT EXISTS bucket_id s3region; +ALTER TYPE OBJECT_TYPE ADD VALUE 'mldata'; ALTER TYPE s3region ADD VALUE 'b5'; +ALTER TYPE s3region ADD VALUE 'b6'; -- Create the file_data table CREATE TABLE IF NOT EXISTS file_data ( From 0c392a22a40783701e4b1de41f087c7bb57de082 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:04:13 +0530 Subject: [PATCH 183/211] [server] Rename --- server/configurations/local.yaml | 2 +- server/ente/file.go | 2 +- server/ente/filedata/filedata.go | 6 +++--- server/ente/filedata/path.go | 4 ++-- server/ente/filedata/putfiledata.go | 4 ++-- server/pkg/controller/filedata/controller.go | 2 +- server/pkg/utils/s3config/s3config.go | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/server/configurations/local.yaml b/server/configurations/local.yaml index 1adad9e775..cbe9b949c3 100644 --- a/server/configurations/local.yaml +++ b/server/configurations/local.yaml @@ -174,7 +174,7 @@ s3: #use_path_style_urls: true # # file-data-storage: - # derivedMeta: + # mldata: # primaryBucket: # replicaBuckets: [] # img_preview: diff --git a/server/ente/file.go b/server/ente/file.go index 1e00c7f258..117db44729 100644 --- a/server/ente/file.go +++ b/server/ente/file.go @@ -157,7 +157,7 @@ const ( THUMBNAIL ObjectType = "thumbnail" PreviewImage ObjectType = "img_preview" PreviewVideo ObjectType = "vid_preview" - DerivedMeta ObjectType = "derivedMeta" + MlData ObjectType = "mldata" ) // S3ObjectKey represents the s3 object key and corresponding fileID for it diff --git a/server/ente/filedata/filedata.go b/server/ente/filedata/filedata.go index 9d5f366351..3db26fefb0 100644 --- a/server/ente/filedata/filedata.go +++ b/server/ente/filedata/filedata.go @@ -19,7 +19,7 @@ type GetFilesData struct { } func (g *GetFilesData) Validate() error { - if g.Type != ente.PreviewVideo && g.Type != ente.DerivedMeta { + if g.Type != ente.PreviewVideo && g.Type != ente.MlData { return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) } if len(g.FileIDs) == 0 { @@ -37,7 +37,7 @@ type GetFileData struct { } func (g *GetFileData) Validate() error { - if g.Type != ente.PreviewVideo && g.Type != ente.DerivedMeta { + if g.Type != ente.PreviewVideo && g.Type != ente.MlData { return ente.NewBadRequestWithMessage(fmt.Sprintf("unsupported object type %s", g.Type)) } return nil @@ -102,7 +102,7 @@ type Row struct { // S3FileMetadataObjectKey returns the object key for the metadata stored in the S3 bucket. func (r *Row) S3FileMetadataObjectKey() string { - if r.Type == ente.DerivedMeta { + if r.Type == ente.MlData { return derivedMetaPath(r.FileID, r.UserID) } if r.Type == ente.PreviewVideo { diff --git a/server/ente/filedata/path.go b/server/ente/filedata/path.go index fce63bad09..d0ea908800 100644 --- a/server/ente/filedata/path.go +++ b/server/ente/filedata/path.go @@ -15,7 +15,7 @@ func AllObjects(fileID int64, ownerID int64, oType ente.ObjectType) []string { switch oType { case ente.PreviewVideo: return []string{previewVideoPath(fileID, ownerID), previewVideoPlaylist(fileID, ownerID)} - case ente.DerivedMeta: + case ente.MlData: return []string{derivedMetaPath(fileID, ownerID)} case ente.PreviewImage: return []string{previewImagePath(fileID, ownerID)} @@ -49,5 +49,5 @@ func previewImagePath(fileID int64, ownerID int64) string { } func derivedMetaPath(fileID int64, ownerID int64) string { - return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.DerivedMeta)) + return fmt.Sprintf("%s%s", BasePrefix(fileID, ownerID), string(ente.MlData)) } diff --git a/server/ente/filedata/putfiledata.go b/server/ente/filedata/putfiledata.go index 08ef42ef1b..58394989da 100644 --- a/server/ente/filedata/putfiledata.go +++ b/server/ente/filedata/putfiledata.go @@ -35,7 +35,7 @@ func (r PutFileDataRequest) Validate() error { if !r.isObjectDataPresent() || r.isEncDataPresent() { return ente.NewBadRequestWithMessage("object (only) data is required for preview image") } - case ente.DerivedMeta: + case ente.MlData: if !r.isEncDataPresent() || r.isObjectDataPresent() { return ente.NewBadRequestWithMessage("encryptedData and decryptionHeader (only) are required for derived meta") } @@ -46,7 +46,7 @@ func (r PutFileDataRequest) Validate() error { } func (r PutFileDataRequest) S3FileMetadataObjectKey(ownerID int64) string { - if r.Type == ente.DerivedMeta { + if r.Type == ente.MlData { return derivedMetaPath(r.FileID, ownerID) } if r.Type == ente.PreviewVideo { diff --git a/server/pkg/controller/filedata/controller.go b/server/pkg/controller/filedata/controller.go index dbdb418fa4..d5f400c4fe 100644 --- a/server/pkg/controller/filedata/controller.go +++ b/server/pkg/controller/filedata/controller.go @@ -84,7 +84,7 @@ func (c *Controller) InsertOrUpdate(ctx *gin.Context, req *fileData.PutFileDataR if err != nil { return stacktrace.Propagate(err, "") } - if req.Type != ente.DerivedMeta && req.Type != ente.PreviewVideo { + if req.Type != ente.MlData && req.Type != ente.PreviewVideo { return stacktrace.Propagate(ente.NewBadRequestWithMessage("unsupported object type "+string(req.Type)), "") } fileOwnerID := userID diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index 46ece00ef0..86cf6f11b5 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -171,7 +171,7 @@ func (config *S3Config) GetBucketID(oType ente.ObjectType) string { if config.fileDataConfig.HasConfig(oType) { return config.fileDataConfig.GetPrimaryBucketID(oType) } - if oType == ente.DerivedMeta || oType == ente.PreviewVideo || oType == ente.PreviewImage { + if oType == ente.MlData || oType == ente.PreviewVideo || oType == ente.PreviewImage { return config.derivedStorageDC } panic(fmt.Sprintf("ops not supported for type: %s", oType)) @@ -180,7 +180,7 @@ func (config *S3Config) GetReplicatedBuckets(oType ente.ObjectType) []string { if config.fileDataConfig.HasConfig(oType) { return config.fileDataConfig.GetReplicaBuckets(oType) } - if oType == ente.DerivedMeta || oType == ente.PreviewVideo || oType == ente.PreviewImage { + if oType == ente.MlData || oType == ente.PreviewVideo || oType == ente.PreviewImage { return []string{} } panic(fmt.Sprintf("ops not supported for object type: %s", oType)) From c88dd22e9c603b474dd0687e1ef5a14e8da0b916 Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:07:55 +0530 Subject: [PATCH 184/211] [server] Add bucket b6 --- server/pkg/utils/s3config/s3config.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/pkg/utils/s3config/s3config.go b/server/pkg/utils/s3config/s3config.go index 86cf6f11b5..469d3fb632 100644 --- a/server/pkg/utils/s3config/s3config.go +++ b/server/pkg/utils/s3config/s3config.go @@ -85,6 +85,7 @@ var ( dcSCWEuropeFrance_v3 string = "scw-eu-fr-v3" dcWasabiEuropeCentralDerived string = "wasabi-eu-central-2-derived" bucket5 string = "b5" + bucket6 string = "b6" ) // Number of days that the wasabi bucket is configured to retain objects. @@ -99,9 +100,9 @@ func NewS3Config() *S3Config { } func (config *S3Config) initialize() { - dcs := [7]string{ + dcs := [8]string{ dcB2EuropeCentral, dcSCWEuropeFranceLockedDeprecated, dcWasabiEuropeCentralDeprecated, - dcWasabiEuropeCentral_v3, dcSCWEuropeFrance_v3, dcWasabiEuropeCentralDerived, bucket5} + dcWasabiEuropeCentral_v3, dcSCWEuropeFrance_v3, dcWasabiEuropeCentralDerived, bucket5, bucket6} config.hotDC = dcB2EuropeCentral config.secondaryHotDC = dcWasabiEuropeCentral_v3 From 3502fcac5e14abb55c66e29696b3c7f78bd4bf5e Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 13:10:15 +0530 Subject: [PATCH 185/211] Move strings to translations --- .../src/components/Sidebar/Preferences.tsx | 3 +-- .../base/locales/en-US/translation.json | 11 +++++++++ .../new/photos/components/MLSettings.tsx | 24 +++++++------------ 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/web/apps/photos/src/components/Sidebar/Preferences.tsx b/web/apps/photos/src/components/Sidebar/Preferences.tsx index 05d72c92ae..69671791ae 100644 --- a/web/apps/photos/src/components/Sidebar/Preferences.tsx +++ b/web/apps/photos/src/components/Sidebar/Preferences.tsx @@ -4,7 +4,6 @@ import { MenuItemGroup, MenuSectionTitle } from "@/base/components/Menu"; import { Titlebar } from "@/base/components/Titlebar"; import { getLocaleInUse, - pt, setLocaleInUse, supportedLocales, type SupportedLocale, @@ -96,7 +95,7 @@ export const Preferences: React.FC = ({ } onClick={() => setOpenMLSettings(true)} - label={pt("Face and magic search")} + label={t("face_and_magic_search")} /> diff --git a/web/packages/base/locales/en-US/translation.json b/web/packages/base/locales/en-US/translation.json index 1adafa2a79..df7047d359 100644 --- a/web/packages/base/locales/en-US/translation.json +++ b/web/packages/base/locales/en-US/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "Scan QR code instead", "ENABLE_TWO_FACTOR": "Enable two-factor", "ENABLE": "Enable", + "enabled": "Enabled", "LOST_DEVICE": "Lost two-factor device", "INCORRECT_CODE": "Incorrect code", "TWO_FACTOR_INFO": "Add an additional layer of security by requiring more than your email and password to log in to your account", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

You have dragged and dropped a mixture of files and folders.

Please provide either only files, or only folders when selecting option to create separate albums

", "CHOSE_THEME": "Choose theme", "more_details": "More details", + "ml_search": "Face and magic search", + "ml_search_description": "Ente supports on-device machine learning for face recognition, magic search and other advanced search features", + "ml_search_footnote": "Magic search allows to search photos by their contents, e.g. 'car', 'red car', 'Ferrari'", + "indexing": "Indexing", + "processed": "Processed", + "indexing_status_running": "Running", + "indexing_status_scheduled": "Scheduled", + "indexing_status_done": "Done", + "ml_search_disable": "Disable face and magic search", + "ml_search_disable_confirm": "Do you want to disable face and magic search on all your devices?", "ENABLE_FACE_SEARCH": "Enable face recognition", "ENABLE_FACE_SEARCH_TITLE": "Enable face recognition?", "ENABLE_FACE_SEARCH_DESCRIPTION": "

If you enable face recognition, Ente will extract face geometry from your photos. This will happen on your device, and any generated biometric data will be end-to-encrypted.

Please click here for more details about this feature in our privacy policy

", diff --git a/web/packages/new/photos/components/MLSettings.tsx b/web/packages/new/photos/components/MLSettings.tsx index d36067155e..bce2617dba 100644 --- a/web/packages/new/photos/components/MLSettings.tsx +++ b/web/packages/new/photos/components/MLSettings.tsx @@ -124,7 +124,7 @@ export const MLSettings: React.FC = ({ {component} @@ -161,9 +161,7 @@ const EnableML: React.FC = ({ onEnable }) => { return ( - {pt( - "Ente supports on-device machine learning for face recognition, magic search and other advanced search features", - )} + {t("ml_search_description")} - {pt( - 'Magic search allows to search photos by their contents, e.g. "car", "red car", "Ferrari"', - )} + {t("ml_search_footnote")} ); @@ -319,14 +315,12 @@ const ManageML: React.FC = ({ const confirmDisableML = () => { setDialogBoxAttributesV2({ - title: pt("Disable face and magic search"), - content: pt( - "Do you want to disable face and magic search on all your devices?", - ), + title: t("ml_search_disable"), + content: t("ml_search_disable_confirm"), close: { text: t("cancel") }, proceed: { variant: "critical", - text: pt("Disable"), + text: t("DISABLE"), action: onDisableML, }, buttonDirection: "row", @@ -338,7 +332,7 @@ const ManageML: React.FC = ({ = ({ justifyContent={"space-between"} > - {pt("Indexing")} + {t("indexing")} {status} @@ -370,7 +364,7 @@ const ManageML: React.FC = ({ justifyContent={"space-between"} > - {pt("Processed")} + {t("processed")} {processed} From 1564d9c0cac9e60251d84c98f7af85290f65a04e Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:10:54 +0530 Subject: [PATCH 186/211] refactor --- server/pkg/utils/s3config/filedata.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/server/pkg/utils/s3config/filedata.go b/server/pkg/utils/s3config/filedata.go index fca1ecbcf3..5f70cba043 100644 --- a/server/pkg/utils/s3config/filedata.go +++ b/server/pkg/utils/s3config/filedata.go @@ -17,25 +17,29 @@ type FileDataConfig struct { func (f FileDataConfig) HasConfig(objectType ente.ObjectType) bool { if objectType == "" || objectType == ente.FILE || objectType == ente.THUMBNAIL { - panic(fmt.Sprintf("Invalid object type: %s", objectType)) + panic(fmt.Sprintf("Unsupported object type: %s", objectType)) } - _, ok := f.ObjectBucketConfig[strings.ToLower(string(objectType))] + _, ok := f.ObjectBucketConfig[key(objectType)] return ok } func (f FileDataConfig) GetPrimaryBucketID(objectType ente.ObjectType) string { - config, ok := f.ObjectBucketConfig[strings.ToLower(string(objectType))] + config, ok := f.ObjectBucketConfig[key(objectType)] if !ok { - panic(fmt.Sprintf("No config for object type: %s, use HasConfig", objectType)) + panic(fmt.Sprintf("No config for object type: %s, use HasConfig", key(objectType))) } return config.PrimaryBucket } func (f FileDataConfig) GetReplicaBuckets(objectType ente.ObjectType) []string { - config, ok := f.ObjectBucketConfig[strings.ToLower(string(objectType))] + config, ok := f.ObjectBucketConfig[key(objectType)] if !ok { - panic(fmt.Sprintf("No config for object type: %s, use HasConfig", objectType)) + panic(fmt.Sprintf("No config for object type: %s, use HasConfig", key(objectType))) } return config.ReplicaBuckets } + +func key(oType ente.ObjectType) string { + return strings.ToLower(string(oType)) +} From a6c5d0328613cf9033afff9d247e9d83b3447f7c Mon Sep 17 00:00:00 2001 From: Neeraj Gupta <254676+ua741@users.noreply.github.com> Date: Mon, 12 Aug 2024 13:21:11 +0530 Subject: [PATCH 187/211] rename --- server/pkg/api/file.go | 80 ++++++++++++++++++------------------- server/pkg/api/file_data.go | 20 +++++----- 2 files changed, 50 insertions(+), 50 deletions(-) diff --git a/server/pkg/api/file.go b/server/pkg/api/file.go index 65efb1d068..2e15ade325 100644 --- a/server/pkg/api/file.go +++ b/server/pkg/api/file.go @@ -33,7 +33,7 @@ const DefaultMaxBatchSize = 1000 const DefaultCopyBatchSize = 100 // CreateOrUpdate creates an entry for a file -func (f *FileHandler) CreateOrUpdate(c *gin.Context) { +func (h *FileHandler) CreateOrUpdate(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) var file ente.File if err := c.ShouldBindJSON(&file); err != nil { @@ -48,7 +48,7 @@ func (f *FileHandler) CreateOrUpdate(c *gin.Context) { if file.ID == 0 { file.OwnerID = userID file.IsDeleted = false - file, err := f.Controller.Create(c, userID, file, c.Request.UserAgent(), enteApp) + file, err := h.Controller.Create(c, userID, file, c.Request.UserAgent(), enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -56,7 +56,7 @@ func (f *FileHandler) CreateOrUpdate(c *gin.Context) { c.JSON(http.StatusOK, file) return } - response, err := f.Controller.Update(c, userID, file, enteApp) + response, err := h.Controller.Update(c, userID, file, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -65,7 +65,7 @@ func (f *FileHandler) CreateOrUpdate(c *gin.Context) { } // CopyFiles copies files that are owned by another user -func (f *FileHandler) CopyFiles(c *gin.Context) { +func (h *FileHandler) CopyFiles(c *gin.Context) { var req ente.CopyFileSyncRequest if err := c.ShouldBindJSON(&req); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -75,7 +75,7 @@ func (f *FileHandler) CopyFiles(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.NewBadRequestWithMessage(fmt.Sprintf("more than %d items", DefaultCopyBatchSize)), "")) return } - response, err := f.FileCopyCtrl.CopyFiles(c, req) + response, err := h.FileCopyCtrl.CopyFiles(c, req) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -84,7 +84,7 @@ func (f *FileHandler) CopyFiles(c *gin.Context) { } // Update updates already existing file -func (f *FileHandler) Update(c *gin.Context) { +func (h *FileHandler) Update(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) @@ -98,7 +98,7 @@ func (f *FileHandler) Update(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, "fileID should be >0")) return } - response, err := f.Controller.Update(c, userID, file, enteApp) + response, err := h.Controller.Update(c, userID, file, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -107,12 +107,12 @@ func (f *FileHandler) Update(c *gin.Context) { } // GetUploadURLs returns a bunch of urls where in the user can upload objects -func (f *FileHandler) GetUploadURLs(c *gin.Context) { +func (h *FileHandler) GetUploadURLs(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) count, _ := strconv.Atoi(c.Query("count")) - urls, err := f.Controller.GetUploadURLs(c, userID, count, enteApp, false) + urls, err := h.Controller.GetUploadURLs(c, userID, count, enteApp, false) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -123,12 +123,12 @@ func (f *FileHandler) GetUploadURLs(c *gin.Context) { } // GetMultipartUploadURLs returns an array of PartUpload PresignedURLs -func (f *FileHandler) GetMultipartUploadURLs(c *gin.Context) { +func (h *FileHandler) GetMultipartUploadURLs(c *gin.Context) { enteApp := auth.GetApp(c) userID := auth.GetUserID(c.Request.Header) count, _ := strconv.Atoi(c.Query("count")) - urls, err := f.Controller.GetMultipartUploadURLs(c, userID, count, enteApp) + urls, err := h.Controller.GetMultipartUploadURLs(c, userID, count, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -139,21 +139,21 @@ func (f *FileHandler) GetMultipartUploadURLs(c *gin.Context) { } // Get redirects the request to the file location -func (f *FileHandler) Get(c *gin.Context) { +func (h *FileHandler) Get(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := f.Controller.GetFileURL(c, userID, fileID) + url, err := h.Controller.GetFileURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - f.logBadRedirect(c) + h.logBadRedirect(c) c.Redirect(http.StatusTemporaryRedirect, url) } // GetV2 returns the URL of the file to client -func (f *FileHandler) GetV2(c *gin.Context) { +func (h *FileHandler) GetV2(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := f.Controller.GetFileURL(c, userID, fileID) + url, err := h.Controller.GetFileURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -164,21 +164,21 @@ func (f *FileHandler) GetV2(c *gin.Context) { } // GetThumbnail redirects the request to the file's thumbnail location -func (f *FileHandler) GetThumbnail(c *gin.Context) { +func (h *FileHandler) GetThumbnail(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := f.Controller.GetThumbnailURL(c, userID, fileID) + url, err := h.Controller.GetThumbnailURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - f.logBadRedirect(c) + h.logBadRedirect(c) c.Redirect(http.StatusTemporaryRedirect, url) } // GetThumbnailV2 returns the URL of the thumbnail to the client -func (f *FileHandler) GetThumbnailV2(c *gin.Context) { +func (h *FileHandler) GetThumbnailV2(c *gin.Context) { userID, fileID := getUserAndFileIDs(c) - url, err := f.Controller.GetThumbnailURL(c, userID, fileID) + url, err := h.Controller.GetThumbnailURL(c, userID, fileID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -189,7 +189,7 @@ func (f *FileHandler) GetThumbnailV2(c *gin.Context) { } // Trash moves the given files to the trash bin -func (f *FileHandler) Trash(c *gin.Context) { +func (h *FileHandler) Trash(c *gin.Context) { var request ente.TrashRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "failed to bind")) @@ -201,7 +201,7 @@ func (f *FileHandler) Trash(c *gin.Context) { } userID := auth.GetUserID(c.Request.Header) request.OwnerID = userID - err := f.Controller.Trash(c, userID, request) + err := h.Controller.Trash(c, userID, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -210,7 +210,7 @@ func (f *FileHandler) Trash(c *gin.Context) { } // GetSize returns the size of files indicated by fileIDs -func (f *FileHandler) GetSize(c *gin.Context) { +func (h *FileHandler) GetSize(c *gin.Context) { var request ente.FileIDsRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -227,7 +227,7 @@ func (f *FileHandler) GetSize(c *gin.Context) { return } - size, err := f.Controller.GetSize(userID, request.FileIDs) + size, err := h.Controller.GetSize(userID, request.FileIDs) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -238,7 +238,7 @@ func (f *FileHandler) GetSize(c *gin.Context) { } // GetInfo returns the FileInfo of files indicated by fileIDs -func (f *FileHandler) GetInfo(c *gin.Context) { +func (h *FileHandler) GetInfo(c *gin.Context) { var request ente.FileIDsRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "failed to bind request")) @@ -246,7 +246,7 @@ func (f *FileHandler) GetInfo(c *gin.Context) { } userID := auth.GetUserID(c.Request.Header) - response, err := f.Controller.GetFileInfo(c, userID, request.FileIDs) + response, err := h.Controller.GetFileInfo(c, userID, request.FileIDs) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) } else { @@ -295,9 +295,9 @@ func shouldRejectRequest(c *gin.Context) (bool, error) { } // GetDuplicates returns the list of files of the same size -func (f *FileHandler) GetDuplicates(c *gin.Context) { +func (h *FileHandler) GetDuplicates(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) - dupes, err := f.Controller.GetDuplicates(userID) + dupes, err := h.Controller.GetDuplicates(userID) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -308,10 +308,10 @@ func (f *FileHandler) GetDuplicates(c *gin.Context) { } // GetLargeThumbnail returns the list of files whose thumbnail size is larger than threshold size -func (f *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { +func (h *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { userID := auth.GetUserID(c.Request.Header) threshold, _ := strconv.ParseInt(c.Query("threshold"), 10, 64) - largeThumbnailFiles, err := f.Controller.GetLargeThumbnailFiles(userID, threshold) + largeThumbnailFiles, err := h.Controller.GetLargeThumbnailFiles(userID, threshold) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -322,7 +322,7 @@ func (f *FileHandler) GetLargeThumbnailFiles(c *gin.Context) { } // UpdateMagicMetadata updates magic metadata for a list of files. -func (f *FileHandler) UpdateMagicMetadata(c *gin.Context) { +func (h *FileHandler) UpdateMagicMetadata(c *gin.Context) { var request ente.UpdateMultipleMagicMetadataRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) @@ -332,7 +332,7 @@ func (f *FileHandler) UpdateMagicMetadata(c *gin.Context) { handler.Error(c, stacktrace.Propagate(ente.ErrBatchSizeTooLarge, "")) return } - err := f.Controller.UpdateMagicMetadata(c, request, false) + err := h.Controller.UpdateMagicMetadata(c, request, false) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -341,13 +341,13 @@ func (f *FileHandler) UpdateMagicMetadata(c *gin.Context) { } // UpdatePublicMagicMetadata updates public magic metadata for a list of files. -func (f *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { +func (h *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { var request ente.UpdateMultipleMagicMetadataRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return } - err := f.Controller.UpdateMagicMetadata(c, request, true) + err := h.Controller.UpdateMagicMetadata(c, request, true) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -356,7 +356,7 @@ func (f *FileHandler) UpdatePublicMagicMetadata(c *gin.Context) { } // UpdateThumbnail updates thumbnail of a file -func (f *FileHandler) UpdateThumbnail(c *gin.Context) { +func (h *FileHandler) UpdateThumbnail(c *gin.Context) { enteApp := auth.GetApp(c) var request ente.UpdateThumbnailRequest @@ -364,7 +364,7 @@ func (f *FileHandler) UpdateThumbnail(c *gin.Context) { handler.Error(c, stacktrace.Propagate(err, "")) return } - err := f.Controller.UpdateThumbnail(c, request.FileID, request.Thumbnail, enteApp) + err := h.Controller.UpdateThumbnail(c, request.FileID, request.Thumbnail, enteApp) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -372,8 +372,8 @@ func (f *FileHandler) UpdateThumbnail(c *gin.Context) { c.Status(http.StatusOK) } -func (f *FileHandler) GetTotalFileCount(c *gin.Context) { - count, err := f.Controller.GetTotalFileCount() +func (h *FileHandler) GetTotalFileCount(c *gin.Context) { + count, err := h.Controller.GetTotalFileCount() if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -390,7 +390,7 @@ func getUserAndFileIDs(c *gin.Context) (int64, int64) { } // logBadRedirect will log the request id if we are redirecting to another url with the auth-token in header -func (f *FileHandler) logBadRedirect(c *gin.Context) { +func (h *FileHandler) logBadRedirect(c *gin.Context) { if len(c.GetHeader("X-Auth-Token")) != 0 && os.Getenv("ENVIRONMENT") != "" { log.WithField("req_id", requestid.Get(c)).Error("critical: sending token to another service") } diff --git a/server/pkg/api/file_data.go b/server/pkg/api/file_data.go index d45a3392b8..36c863e65c 100644 --- a/server/pkg/api/file_data.go +++ b/server/pkg/api/file_data.go @@ -10,7 +10,7 @@ import ( "net/http" ) -func (f *FileHandler) PutFileData(ctx *gin.Context) { +func (h *FileHandler) PutFileData(ctx *gin.Context) { var req fileData.PutFileDataRequest if err := ctx.ShouldBindJSON(&req); err != nil { ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error())) @@ -25,7 +25,7 @@ func (f *FileHandler) PutFileData(ctx *gin.Context) { version := 1 reqInt.Version = &version } - err := f.FileDataCtrl.InsertOrUpdate(ctx, &req) + err := h.FileDataCtrl.InsertOrUpdate(ctx, &req) if err != nil { handler.Error(ctx, err) @@ -34,13 +34,13 @@ func (f *FileHandler) PutFileData(ctx *gin.Context) { ctx.JSON(http.StatusOK, gin.H{}) } -func (f *FileHandler) GetFilesData(ctx *gin.Context) { +func (h *FileHandler) GetFilesData(ctx *gin.Context) { var req fileData.GetFilesData if err := ctx.ShouldBindJSON(&req); err != nil { ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error())) return } - resp, err := f.FileDataCtrl.GetFilesData(ctx, req) + resp, err := h.FileDataCtrl.GetFilesData(ctx, req) if err != nil { handler.Error(ctx, err) return @@ -48,13 +48,13 @@ func (f *FileHandler) GetFilesData(ctx *gin.Context) { ctx.JSON(http.StatusOK, resp) } -func (f *FileHandler) GetFileData(ctx *gin.Context) { +func (h *FileHandler) GetFileData(ctx *gin.Context) { var req fileData.GetFileData if err := ctx.ShouldBindJSON(&req); err != nil { ctx.JSON(http.StatusBadRequest, ente.NewBadRequestWithMessage(err.Error())) return } - resp, err := f.FileDataCtrl.GetFileData(ctx, req) + resp, err := h.FileDataCtrl.GetFileData(ctx, req) if err != nil { handler.Error(ctx, err) return @@ -64,13 +64,13 @@ func (f *FileHandler) GetFileData(ctx *gin.Context) { }) } -func (f *FileHandler) GetPreviewUploadURL(c *gin.Context) { +func (h *FileHandler) GetPreviewUploadURL(c *gin.Context) { var request fileData.PreviewUploadUrlRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) return } - url, err := f.FileDataCtrl.PreviewUploadURL(c, request) + url, err := h.FileDataCtrl.PreviewUploadURL(c, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return @@ -80,13 +80,13 @@ func (f *FileHandler) GetPreviewUploadURL(c *gin.Context) { }) } -func (f *FileHandler) GetPreviewURL(c *gin.Context) { +func (h *FileHandler) GetPreviewURL(c *gin.Context) { var request fileData.GetPreviewURLRequest if err := c.ShouldBindJSON(&request); err != nil { handler.Error(c, stacktrace.Propagate(ente.ErrBadRequest, fmt.Sprintf("Request binding failed %s", err))) return } - url, err := f.FileDataCtrl.GetPreviewUrl(c, request) + url, err := h.FileDataCtrl.GetPreviewUrl(c, request) if err != nil { handler.Error(c, stacktrace.Propagate(err, "")) return From 20fb9e99f0a97d0116664ae51bcfa667af7401de Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 14:02:08 +0530 Subject: [PATCH 188/211] context --- web/packages/base/locales/en-US/translation.json | 2 +- web/packages/new/photos/components/MLSettings.tsx | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/web/packages/base/locales/en-US/translation.json b/web/packages/base/locales/en-US/translation.json index df7047d359..b3f4e796c7 100644 --- a/web/packages/base/locales/en-US/translation.json +++ b/web/packages/base/locales/en-US/translation.json @@ -478,7 +478,7 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

You have dragged and dropped a mixture of files and folders.

Please provide either only files, or only folders when selecting option to create separate albums

", "CHOSE_THEME": "Choose theme", "more_details": "More details", - "ml_search": "Face and magic search", + "face_and_magic_search": "Face and magic search", "ml_search_description": "Ente supports on-device machine learning for face recognition, magic search and other advanced search features", "ml_search_footnote": "Magic search allows to search photos by their contents, e.g. 'car', 'red car', 'Ferrari'", "indexing": "Indexing", diff --git a/web/packages/new/photos/components/MLSettings.tsx b/web/packages/new/photos/components/MLSettings.tsx index bce2617dba..a3d12a41cc 100644 --- a/web/packages/new/photos/components/MLSettings.tsx +++ b/web/packages/new/photos/components/MLSettings.tsx @@ -1,7 +1,6 @@ import { EnteDrawer } from "@/base/components/EnteDrawer"; import { MenuItemGroup } from "@/base/components/Menu"; import { Titlebar } from "@/base/components/Titlebar"; -import { pt } from "@/base/i18n"; import log from "@/base/log"; import { disableML, @@ -301,14 +300,14 @@ const ManageML: React.FC = ({ let status: string; switch (phase) { case "indexing": - status = pt("Running"); + status = t("running"); break; case "scheduled": - status = pt("Scheduled"); + status = t("scheduled"); break; // TODO: Clustering default: - status = pt("Done"); + status = t("done"); break; } const processed = `${nSyncedFiles} / ${nTotalFiles}`; @@ -352,7 +351,9 @@ const ManageML: React.FC = ({ {t("indexing")} - {status} + + {t("indexing_status", { context: status })} + Date: Mon, 12 Aug 2024 14:07:08 +0530 Subject: [PATCH 189/211] [mob][photos] Auto-fill password for Applock --- mobile/lib/ui/components/text_input_widget.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mobile/lib/ui/components/text_input_widget.dart b/mobile/lib/ui/components/text_input_widget.dart index 0d4fd7aeca..84ef249b03 100644 --- a/mobile/lib/ui/components/text_input_widget.dart +++ b/mobile/lib/ui/components/text_input_widget.dart @@ -156,6 +156,8 @@ class _TextInputWidgetState extends State { keyboardType: widget.textInputType, textCapitalization: widget.textCapitalization!, autofocus: widget.autoFocus ?? false, + autofillHints: + widget.isPasswordInput ? [AutofillHints.password] : [], controller: _textController, focusNode: widget.focusNode, inputFormatters: widget.textInputFormatter ?? From e8d804468e2f1c7029c460b750033c0ae7c54796 Mon Sep 17 00:00:00 2001 From: Crowdin Bot Date: Mon, 12 Aug 2024 08:53:59 +0000 Subject: [PATCH 190/211] New Crowdin translations by GitHub Action --- web/packages/base/locales/ar-SA/translation.json | 11 +++++++++++ web/packages/base/locales/bg-BG/translation.json | 11 +++++++++++ web/packages/base/locales/ca-ES/translation.json | 11 +++++++++++ web/packages/base/locales/de-DE/translation.json | 11 +++++++++++ web/packages/base/locales/el-GR/translation.json | 11 +++++++++++ web/packages/base/locales/es-ES/translation.json | 11 +++++++++++ web/packages/base/locales/fa-IR/translation.json | 11 +++++++++++ web/packages/base/locales/fi-FI/translation.json | 11 +++++++++++ web/packages/base/locales/fr-FR/translation.json | 11 +++++++++++ web/packages/base/locales/gu-IN/translation.json | 11 +++++++++++ web/packages/base/locales/hi-IN/translation.json | 11 +++++++++++ web/packages/base/locales/id-ID/translation.json | 11 +++++++++++ web/packages/base/locales/is-IS/translation.json | 11 +++++++++++ web/packages/base/locales/it-IT/translation.json | 11 +++++++++++ web/packages/base/locales/ja-JP/translation.json | 11 +++++++++++ web/packages/base/locales/ko-KR/translation.json | 11 +++++++++++ web/packages/base/locales/nl-NL/translation.json | 11 +++++++++++ web/packages/base/locales/pl-PL/translation.json | 11 +++++++++++ web/packages/base/locales/pt-BR/translation.json | 11 +++++++++++ web/packages/base/locales/pt-PT/translation.json | 11 +++++++++++ web/packages/base/locales/ru-RU/translation.json | 11 +++++++++++ web/packages/base/locales/sv-SE/translation.json | 11 +++++++++++ web/packages/base/locales/te-IN/translation.json | 11 +++++++++++ web/packages/base/locales/th-TH/translation.json | 11 +++++++++++ web/packages/base/locales/ti-ER/translation.json | 11 +++++++++++ web/packages/base/locales/tr-TR/translation.json | 11 +++++++++++ web/packages/base/locales/zh-CN/translation.json | 11 +++++++++++ 27 files changed, 297 insertions(+) diff --git a/web/packages/base/locales/ar-SA/translation.json b/web/packages/base/locales/ar-SA/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/ar-SA/translation.json +++ b/web/packages/base/locales/ar-SA/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/bg-BG/translation.json b/web/packages/base/locales/bg-BG/translation.json index e3332b04ba..06892c534a 100644 --- a/web/packages/base/locales/bg-BG/translation.json +++ b/web/packages/base/locales/bg-BG/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/ca-ES/translation.json b/web/packages/base/locales/ca-ES/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/ca-ES/translation.json +++ b/web/packages/base/locales/ca-ES/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/de-DE/translation.json b/web/packages/base/locales/de-DE/translation.json index 4c6877426f..c2b924a90d 100644 --- a/web/packages/base/locales/de-DE/translation.json +++ b/web/packages/base/locales/de-DE/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "QR‐Code stattdessen scannen", "ENABLE_TWO_FACTOR": "Zwei-Faktor-Authentifizierung aktivieren", "ENABLE": "Aktivieren", + "enabled": "", "LOST_DEVICE": "Zwei-Faktor-Gerät verloren", "INCORRECT_CODE": "Falscher Code", "TWO_FACTOR_INFO": "Fügen Sie eine zusätzliche Sicherheitsebene hinzu, indem Sie mehr als Ihre E-Mail und Ihr Passwort benötigen, um sich mit Ihrem Account anzumelden", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

Du hast sowohl Dateien als auch Ordner in das Ente-Fenster gezogen.

Bitte wähle entweder nur Dateien oder nur Ordner aus, wenn separate Alben erstellt werden sollen

", "CHOSE_THEME": "Design auswählen", "more_details": "Weitere Details", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Gesichtserkennung aktivieren", "ENABLE_FACE_SEARCH_TITLE": "Gesichtserkennung aktivieren?", "ENABLE_FACE_SEARCH_DESCRIPTION": "

Wenn du die Gesichtserkennung aktivierst, wird Ente Gesichtsgeometrie aus deinen Fotos extrahieren. Dies wird auf deinem Gerät geschehen, und alle erzeugten biometrischen Daten werden Ende-zu-verschlüsselt.

Bitte klicke hier für weitere Informationen über diese Funktion in unserer Datenschutzerklärung

", diff --git a/web/packages/base/locales/el-GR/translation.json b/web/packages/base/locales/el-GR/translation.json index 4f00dd9620..234f9c1874 100644 --- a/web/packages/base/locales/el-GR/translation.json +++ b/web/packages/base/locales/el-GR/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "Ενεργοποίηση", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "Εσφαλμένος κωδικός", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "Επιλογή θέματος", "more_details": "Περισσότερες λεπτομέρειες", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Ενεργοποίηση αναγνώρισης προσώπου", "ENABLE_FACE_SEARCH_TITLE": "Ενεργοποίηση αναγνώρισης προσώπου;", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/es-ES/translation.json b/web/packages/base/locales/es-ES/translation.json index 4d6090e7f1..cc9d0b8f53 100644 --- a/web/packages/base/locales/es-ES/translation.json +++ b/web/packages/base/locales/es-ES/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "Escanear código QR en su lugar", "ENABLE_TWO_FACTOR": "Activar dos factores", "ENABLE": "Activar", + "enabled": "", "LOST_DEVICE": "Perdido el dispositivo de doble factor", "INCORRECT_CODE": "Código incorrecto", "TWO_FACTOR_INFO": "Añade una capa adicional de seguridad al requerir más de tu email y contraseña para iniciar sesión en tu cuenta", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

Has arrastrado y soltado una mezcla de archivos y carpetas.

Por favor proporcione sólo archivos o carpetas cuando seleccione la opción de crear álbumes separados

", "CHOSE_THEME": "Elegir tema", "more_details": "Más detalles", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Activar búsqueda facial", "ENABLE_FACE_SEARCH_TITLE": "Activar búsqueda facial?", "ENABLE_FACE_SEARCH_DESCRIPTION": "

Si activas la búsqueda facial, ente extraerá la geometría facial de tus fotos. Esto sucederá en su dispositivo y cualquier dato biométrico generado será cifrado de extremo a extremo.

Haga clic aquí para obtener más detalles sobre esta característica en nuestra política de privacidad

", diff --git a/web/packages/base/locales/fa-IR/translation.json b/web/packages/base/locales/fa-IR/translation.json index 7b64b0b8c2..e75d10e3db 100644 --- a/web/packages/base/locales/fa-IR/translation.json +++ b/web/packages/base/locales/fa-IR/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/fi-FI/translation.json b/web/packages/base/locales/fi-FI/translation.json index 7339a684b5..85d6e9c46e 100644 --- a/web/packages/base/locales/fi-FI/translation.json +++ b/web/packages/base/locales/fi-FI/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/fr-FR/translation.json b/web/packages/base/locales/fr-FR/translation.json index 9600ac8ea4..9ae20c4d51 100644 --- a/web/packages/base/locales/fr-FR/translation.json +++ b/web/packages/base/locales/fr-FR/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "Scannez le QRCode de préférence", "ENABLE_TWO_FACTOR": "Activer la double-authentification", "ENABLE": "Activer", + "enabled": "", "LOST_DEVICE": "Perte de l'appareil identificateur", "INCORRECT_CODE": "Code non valide", "TWO_FACTOR_INFO": "Rajoutez une couche de sécurité supplémentaire afin de pas utiliser simplement votre e-mail et mot de passe pour vous connecter à votre compte", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

Vous avez glissé déposé un mélange de fichiers et dossiers.

Veuillez sélectionner soit uniquement des fichiers, ou des dossiers lors du choix d'options pour créer des albums séparés

", "CHOSE_THEME": "Choisir un thème", "more_details": "Plus de détails", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Activer la recherche faciale", "ENABLE_FACE_SEARCH_TITLE": "Activer la recherche faciale ?", "ENABLE_FACE_SEARCH_DESCRIPTION": "

If you enable face search, Ente will extract face geometry from your photos. This will happen on your device, and any generated biometric data will be end-to-encrypted.

Please click here for more details about this feature in our privacy policy

", diff --git a/web/packages/base/locales/gu-IN/translation.json b/web/packages/base/locales/gu-IN/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/gu-IN/translation.json +++ b/web/packages/base/locales/gu-IN/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/hi-IN/translation.json b/web/packages/base/locales/hi-IN/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/hi-IN/translation.json +++ b/web/packages/base/locales/hi-IN/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/id-ID/translation.json b/web/packages/base/locales/id-ID/translation.json index 7a3dfdcf77..2caab67e55 100644 --- a/web/packages/base/locales/id-ID/translation.json +++ b/web/packages/base/locales/id-ID/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "Aktifkan autentikasi dua langkah", "ENABLE": "Aktifkan", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "Kode salah", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "Pilih tema", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Aktifkan pengenalan wajah", "ENABLE_FACE_SEARCH_TITLE": "Aktifkan pengenalan wajah?", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/is-IS/translation.json b/web/packages/base/locales/is-IS/translation.json index 0af34f6a6d..0741125758 100644 --- a/web/packages/base/locales/is-IS/translation.json +++ b/web/packages/base/locales/is-IS/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/it-IT/translation.json b/web/packages/base/locales/it-IT/translation.json index 32d22dff1d..1cd8f128b9 100644 --- a/web/packages/base/locales/it-IT/translation.json +++ b/web/packages/base/locales/it-IT/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "Oppure scansiona il codice QR", "ENABLE_TWO_FACTOR": "Attiva due fattori", "ENABLE": "Attiva", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "Codice errato", "TWO_FACTOR_INFO": "Aggiungi un ulteriore livello di sicurezza richiedendo più informazioni rispetto a email e password per eseguire l'accesso al tuo account", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "Seleziona tema", "more_details": "Più dettagli", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/ja-JP/translation.json b/web/packages/base/locales/ja-JP/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/ja-JP/translation.json +++ b/web/packages/base/locales/ja-JP/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/ko-KR/translation.json b/web/packages/base/locales/ko-KR/translation.json index 16b15f3b7a..22dc0da971 100644 --- a/web/packages/base/locales/ko-KR/translation.json +++ b/web/packages/base/locales/ko-KR/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/nl-NL/translation.json b/web/packages/base/locales/nl-NL/translation.json index 89abea4894..2faa3c5bda 100644 --- a/web/packages/base/locales/nl-NL/translation.json +++ b/web/packages/base/locales/nl-NL/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "Scan QR-code in plaats daarvan", "ENABLE_TWO_FACTOR": "Tweestapsverificatie inschakelen", "ENABLE": "Inschakelen", + "enabled": "", "LOST_DEVICE": "Tweestapsverificatie apparaat verloren", "INCORRECT_CODE": "Onjuiste code", "TWO_FACTOR_INFO": "Voeg een extra beveiligingslaag toe door meer dan uw e-mailadres en wachtwoord te vereisen om in te loggen op uw account", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

Je hebt een mix van bestanden en mappen gesleept en laten vallen.

Geef ofwel alleen bestanden aan, of alleen mappen bij het selecteren van de optie om afzonderlijke albums te maken

", "CHOSE_THEME": "Kies thema", "more_details": "Meer details", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Zoeken op gezichten inschakelen", "ENABLE_FACE_SEARCH_TITLE": "Zoeken op gezichten inschakelen?", "ENABLE_FACE_SEARCH_DESCRIPTION": "

Als u zoeken op gezichten inschakelt, analyseert Ente de gezichtsgeometrie uit uw foto's. Dit gebeurt op uw apparaat en alle gegenereerde biometrische gegevens worden end-to-end versleuteld.

Klik hier voor meer informatie over deze functie in ons privacybeleid

", diff --git a/web/packages/base/locales/pl-PL/translation.json b/web/packages/base/locales/pl-PL/translation.json index 26c808e132..02d9d8b8dc 100644 --- a/web/packages/base/locales/pl-PL/translation.json +++ b/web/packages/base/locales/pl-PL/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "Zamiast tego zeskanuj kod QR", "ENABLE_TWO_FACTOR": "Włącz uwierzytelnianie dwustopniowe", "ENABLE": "Włącz", + "enabled": "", "LOST_DEVICE": "Utracono urządzenie dwustopniowe", "INCORRECT_CODE": "Nieprawidłowy kod", "TWO_FACTOR_INFO": "Dodaj dodatkową warstwę bezpieczeństwa, wymagając więcej niż Twojego adresu e-mail i hasła, aby zalogować się na swoje konto", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

Przeciągnąłeś i upuściłeś mieszankę plików i folderów.

Prosimy podać tylko pliki lub tylko foldery podczas wybierania opcji tworzenia oddzielnych albumów

", "CHOSE_THEME": "Wybierz motyw", "more_details": "Więcej szczegółów", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Włącz rozpoznawanie twarzy", "ENABLE_FACE_SEARCH_TITLE": "Włączyć rozpoznawanie twarzy?", "ENABLE_FACE_SEARCH_DESCRIPTION": "

Jeśli włączysz rozpoznawanie twarzy, Ente wyodrębni geometrię twarzy ze zdjęć. Będzie to miało miejsce na Twoim urządzeniu, a wszystkie wygenerowane dane biometryczne będą zaszyfrowane metodą end-to-end.

Kliknij tutaj, aby uzyskać więcej informacji na temat tej funkcji w naszej polityce prywatności

", diff --git a/web/packages/base/locales/pt-BR/translation.json b/web/packages/base/locales/pt-BR/translation.json index 9dc7cf0076..3b9ff6156d 100644 --- a/web/packages/base/locales/pt-BR/translation.json +++ b/web/packages/base/locales/pt-BR/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "Em vez disso, escaneie um Código QR", "ENABLE_TWO_FACTOR": "Ativar autenticação de dois fatores", "ENABLE": "Habilitar", + "enabled": "", "LOST_DEVICE": "Dispositivo de dois fatores perdido", "INCORRECT_CODE": "Código incorreto", "TWO_FACTOR_INFO": "Adicione uma camada adicional de segurança, exigindo mais do que seu e-mail e senha para entrar na sua conta", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

Você arrastou e deixou uma mistura de arquivos e pastas.

Por favor, forneça apenas arquivos ou apenas pastas ao selecionar a opção para criar álbuns separados

", "CHOSE_THEME": "Escolher tema", "more_details": "Mais detalhes", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Habilitar reconhecimento facial", "ENABLE_FACE_SEARCH_TITLE": "Habilitar reconhecimento facial?", "ENABLE_FACE_SEARCH_DESCRIPTION": "

Se você habilitar o reconhecimento facial, o aplicativo extrairá a geometria do rosto de suas fotos. Isso ocorrerá em seu dispositivo, e quaisquer dados biométricos gerados serão criptografados de ponta a ponta.

Por favor, clique aqui para obter mais detalhes sobre esta funcionalidade em nossa política de privacidade

", diff --git a/web/packages/base/locales/pt-PT/translation.json b/web/packages/base/locales/pt-PT/translation.json index ab3c3a2d75..811552fe21 100644 --- a/web/packages/base/locales/pt-PT/translation.json +++ b/web/packages/base/locales/pt-PT/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/ru-RU/translation.json b/web/packages/base/locales/ru-RU/translation.json index daaf4c479e..7018e63b61 100644 --- a/web/packages/base/locales/ru-RU/translation.json +++ b/web/packages/base/locales/ru-RU/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "Сканировать QR-код вместо", "ENABLE_TWO_FACTOR": "Включить двухфакторную аутентификацию", "ENABLE": "Включить", + "enabled": "", "LOST_DEVICE": "Потеряно двухфакторное устройство", "INCORRECT_CODE": "Неверный код", "TWO_FACTOR_INFO": "Добавьте дополнительный уровень безопасности, запросив для входа в свою учетную запись не только адрес электронной почты и пароль", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

Вы перетащили несколько файлов и папок.

Пожалуйста, указывайте либо только файлы, либо только папки при выборе опции создания отдельных альбомов

", "CHOSE_THEME": "Выберите тему", "more_details": "Более подробная информация", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "Включить распознавание лиц", "ENABLE_FACE_SEARCH_TITLE": "Включить распознавание лиц?", "ENABLE_FACE_SEARCH_DESCRIPTION": "

Если вы включите функцию распознавания лиц, Ente извлечет геометрию лица из ваших фотографий. Это произойдет на вашем устройстве, и все сгенерированные биометрические данные будут зашифрованы полностью.

Пожалуйста, нажмите здесь для получения более подробной информации об этой функции в нашей политике конфиденциальности

", diff --git a/web/packages/base/locales/sv-SE/translation.json b/web/packages/base/locales/sv-SE/translation.json index c501930568..45ad964f89 100644 --- a/web/packages/base/locales/sv-SE/translation.json +++ b/web/packages/base/locales/sv-SE/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "Aktivera", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "Felaktig kod", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/te-IN/translation.json b/web/packages/base/locales/te-IN/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/te-IN/translation.json +++ b/web/packages/base/locales/te-IN/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/th-TH/translation.json b/web/packages/base/locales/th-TH/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/th-TH/translation.json +++ b/web/packages/base/locales/th-TH/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/ti-ER/translation.json b/web/packages/base/locales/ti-ER/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/ti-ER/translation.json +++ b/web/packages/base/locales/ti-ER/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/tr-TR/translation.json b/web/packages/base/locales/tr-TR/translation.json index 9b7b0b227f..073ff7b391 100644 --- a/web/packages/base/locales/tr-TR/translation.json +++ b/web/packages/base/locales/tr-TR/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", "ENABLE": "", + "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "", "CHOSE_THEME": "", "more_details": "", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "", "ENABLE_FACE_SEARCH_TITLE": "", "ENABLE_FACE_SEARCH_DESCRIPTION": "", diff --git a/web/packages/base/locales/zh-CN/translation.json b/web/packages/base/locales/zh-CN/translation.json index e56675e887..e0faf85e7e 100644 --- a/web/packages/base/locales/zh-CN/translation.json +++ b/web/packages/base/locales/zh-CN/translation.json @@ -264,6 +264,7 @@ "SCAN_QR_CODE": "改为扫描二维码", "ENABLE_TWO_FACTOR": "启用双重认证", "ENABLE": "启用", + "enabled": "", "LOST_DEVICE": "丢失了双重认证设备", "INCORRECT_CODE": "代码错误", "TWO_FACTOR_INFO": "登录账户时需要的不仅仅是电子邮件和密码，这增加了额外的安全层", @@ -477,6 +478,16 @@ "ROOT_LEVEL_FILE_WITH_FOLDER_NOT_ALLOWED_MESSAGE": "

你已拖放了文件和文件夹的组合。

选择创建单独相册的选项时，请只提供文件或只提供文件夹

", "CHOSE_THEME": "选择主题", "more_details": "更多详情", + "face_and_magic_search": "", + "ml_search_description": "", + "ml_search_footnote": "", + "indexing": "", + "processed": "", + "indexing_status_running": "", + "indexing_status_scheduled": "", + "indexing_status_done": "", + "ml_search_disable": "", + "ml_search_disable_confirm": "", "ENABLE_FACE_SEARCH": "启用面部搜索", "ENABLE_FACE_SEARCH_TITLE": "要启用面部搜索吗？", "ENABLE_FACE_SEARCH_DESCRIPTION": "

如果您启用面部搜索，Ente 将从照片中提取脸部几何形状。 这将发生在您的设备上，任何生成的生物测定数据都将是端到端加密的。

请单击此处以在我们的隐私政策中了解有关此功能的更多详细信息

", From 0c3ef07b3b699fe843562a1efa8a1a3431c5a497 Mon Sep 17 00:00:00 2001 From: Manav Rathi Date: Mon, 12 Aug 2024 14:26:10 +0530 Subject: [PATCH 191/211] [web] Rename tr keys --- .../publicShare/manage/downloadAccess.tsx | 2 +- .../publicShare/manage/linkPassword/index.tsx | 2 +- .../src/components/Sidebar/MapSetting.tsx | 4 ++-- .../src/components/Sidebar/Preferences.tsx | 6 +++--- .../photos/src/components/Sidebar/index.tsx | 2 +- .../src/components/TwoFactor/Modal/Manage.tsx | 6 +++--- web/apps/photos/src/utils/ui/index.tsx | 4 ++-- .../accounts/pages/two-factor/setup.tsx | 2 +- .../base/locales/ar-SA/translation.json | 20 +++++++++---------- .../base/locales/bg-BG/translation.json | 20 +++++++++---------- .../base/locales/ca-ES/translation.json | 20 +++++++++---------- .../base/locales/de-DE/translation.json | 20 +++++++++---------- .../base/locales/el-GR/translation.json | 20 +++++++++---------- .../base/locales/en-US/translation.json | 20 +++++++++---------- .../base/locales/es-ES/translation.json | 20 +++++++++---------- .../base/locales/fa-IR/translation.json | 20 +++++++++---------- .../base/locales/fi-FI/translation.json | 20 +++++++++---------- .../base/locales/fr-FR/translation.json | 20 +++++++++---------- .../base/locales/gu-IN/translation.json | 20 +++++++++---------- .../base/locales/hi-IN/translation.json | 20 +++++++++---------- .../base/locales/id-ID/translation.json | 20 +++++++++---------- .../base/locales/is-IS/translation.json | 20 +++++++++---------- .../base/locales/it-IT/translation.json | 20 +++++++++---------- .../base/locales/ja-JP/translation.json | 20 +++++++++---------- .../base/locales/ko-KR/translation.json | 20 +++++++++---------- .../base/locales/nl-NL/translation.json | 20 +++++++++---------- .../base/locales/pl-PL/translation.json | 20 +++++++++---------- .../base/locales/pt-BR/translation.json | 20 +++++++++---------- .../base/locales/pt-PT/translation.json | 20 +++++++++---------- .../base/locales/ru-RU/translation.json | 20 +++++++++---------- .../base/locales/sv-SE/translation.json | 20 +++++++++---------- .../base/locales/te-IN/translation.json | 20 +++++++++---------- .../base/locales/th-TH/translation.json | 20 +++++++++---------- .../base/locales/ti-ER/translation.json | 20 +++++++++---------- .../base/locales/tr-TR/translation.json | 20 +++++++++---------- .../base/locales/zh-CN/translation.json | 20 +++++++++---------- .../new/photos/components/MLSettings.tsx | 12 +++++------ 37 files changed, 300 insertions(+), 300 deletions(-) diff --git a/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/downloadAccess.tsx b/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/downloadAccess.tsx index 3c88b5f6cd..92008cc75a 100644 --- a/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/downloadAccess.tsx +++ b/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/downloadAccess.tsx @@ -34,7 +34,7 @@ export function ManageDownloadAccess({ content: , close: { text: t("cancel") }, proceed: { - text: t("DISABLE"), + text: t("disable"), action: () => updatePublicShareURLHelper({ collectionID: collection.id, diff --git a/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/index.tsx b/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/index.tsx index 20b2d848d9..2c5890e9ab 100644 --- a/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/index.tsx +++ b/web/apps/photos/src/components/Collections/CollectionShare/publicShare/manage/linkPassword/index.tsx @@ -35,7 +35,7 @@ export function ManageLinkPassword({ content: t("DISABLE_PASSWORD_MESSAGE"), close: { text: t("cancel") }, proceed: { - text: t("DISABLE"), + text: t("disable"), action: () => updatePublicShareURLHelper({ collectionID: collection.id, diff --git a/web/apps/photos/src/components/Sidebar/MapSetting.tsx b/web/apps/photos/src/components/Sidebar/MapSetting.tsx index a64d64c9eb..14817724bb 100644 --- a/web/apps/photos/src/components/Sidebar/MapSetting.tsx +++ b/web/apps/photos/src/components/Sidebar/MapSetting.tsx @@ -189,7 +189,7 @@ function EnableMap({ onClose, enableMap, onRootClose }) { @@ -103,7 +103,7 @@ export default function TwoFactorModalManageSection(props: Iprops) { onClick={warnTwoFactorDisable} size="large" > - {t("DISABLE")} + {t("disable")} diff --git a/web/apps/photos/src/utils/ui/index.tsx b/web/apps/photos/src/utils/ui/index.tsx index 025664e3ac..33bb2b5d60 100644 --- a/web/apps/photos/src/utils/ui/index.tsx +++ b/web/apps/photos/src/utils/ui/index.tsx @@ -154,7 +154,7 @@ export const getMapEnableConfirmationDialog = ( ), proceed: { action: enableMapHelper, - text: t("ENABLE"), + text: t("enable"), variant: "accent", }, close: { text: t("cancel") }, @@ -167,7 +167,7 @@ export const getMapDisableConfirmationDialog = ( content: , proceed: { action: disableMapHelper, - text: t("DISABLE"), + text: t("disable"), variant: "accent", }, close: { text: t("cancel") }, diff --git a/web/packages/accounts/pages/two-factor/setup.tsx b/web/packages/accounts/pages/two-factor/setup.tsx index 9a4bf01782..2fbba6345d 100644 --- a/web/packages/accounts/pages/two-factor/setup.tsx +++ b/web/packages/accounts/pages/two-factor/setup.tsx @@ -74,7 +74,7 @@ const Page: React.FC = () => { {t("GO_BACK")} diff --git a/web/packages/base/locales/ar-SA/translation.json b/web/packages/base/locales/ar-SA/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/ar-SA/translation.json +++ b/web/packages/base/locales/ar-SA/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/bg-BG/translation.json b/web/packages/base/locales/bg-BG/translation.json index 06892c534a..40cd4c7db9 100644 --- a/web/packages/base/locales/bg-BG/translation.json +++ b/web/packages/base/locales/bg-BG/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/ca-ES/translation.json b/web/packages/base/locales/ca-ES/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/ca-ES/translation.json +++ b/web/packages/base/locales/ca-ES/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/de-DE/translation.json b/web/packages/base/locales/de-DE/translation.json index c2b924a90d..968833c50e 100644 --- a/web/packages/base/locales/de-DE/translation.json +++ b/web/packages/base/locales/de-DE/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Bitte gib diesen Code in deiner bevorzugten Authentifizierungs-App ein", "SCAN_QR_CODE": "QR‐Code stattdessen scannen", "ENABLE_TWO_FACTOR": "Zwei-Faktor-Authentifizierung aktivieren", - "ENABLE": "Aktivieren", + "enable": "Aktivieren", "enabled": "", "LOST_DEVICE": "Zwei-Faktor-Gerät verloren", "INCORRECT_CODE": "Falscher Code", "TWO_FACTOR_INFO": "Fügen Sie eine zusätzliche Sicherheitsebene hinzu, indem Sie mehr als Ihre E-Mail und Ihr Passwort benötigen, um sich mit Ihrem Account anzumelden", "DISABLE_TWO_FACTOR_LABEL": "Deaktiviere die Zwei-Faktor-Authentifizierung", "UPDATE_TWO_FACTOR_LABEL": "Authentifizierungsgerät aktualisieren", - "DISABLE": "Deaktivieren", - "RECONFIGURE": "Neu einrichten", + "disable": "Deaktivieren", + "reconfigure": "Neu einrichten", "UPDATE_TWO_FACTOR": "Zweiten Faktor aktualisieren", "UPDATE_TWO_FACTOR_MESSAGE": "Fahren Sie fort, werden alle Ihre zuvor konfigurierten Authentifikatoren ungültig", "UPDATE": "Aktualisierung", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Gesichtserkennung aktivieren", - "ENABLE_FACE_SEARCH_TITLE": "Gesichtserkennung aktivieren?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

Wenn du die Gesichtserkennung aktivierst, wird Ente Gesichtsgeometrie aus deinen Fotos extrahieren. Dies wird auf deinem Gerät geschehen, und alle erzeugten biometrischen Daten werden Ende-zu-verschlüsselt.

Bitte klicke hier für weitere Informationen über diese Funktion in unserer Datenschutzerklärung

", + "enable_face_search": "Gesichtserkennung aktivieren", + "enable_face_search_title": "Gesichtserkennung aktivieren?", + "enable_face_search_description": "

Wenn du die Gesichtserkennung aktivierst, wird Ente Gesichtsgeometrie aus deinen Fotos extrahieren. Dies wird auf deinem Gerät geschehen, und alle erzeugten biometrischen Daten werden Ende-zu-verschlüsselt.

Bitte klicke hier für weitere Informationen über diese Funktion in unserer Datenschutzerklärung

", "ADVANCED": "Erweitert", - "FACE_SEARCH_CONFIRMATION": "Ich verstehe und möchte Ente erlauben, Gesichtsgeometrie zu verarbeiten", - "LABS": "Experimente", + "face_search_confirmation": "Ich verstehe und möchte Ente erlauben, Gesichtsgeometrie zu verarbeiten", + "labs": "Experimente", "YOURS": "von dir", "passphrase_strength_weak": "Passwortstärke: Schwach", "passphrase_strength_moderate": "Passwortstärke: Moderat", "passphrase_strength_strong": "Passwortstärke: Stark", - "PREFERENCES": "Einstellungen", - "LANGUAGE": "Sprache", + "preferences": "Einstellungen", + "language": "Sprache", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Ungültiges Exportverzeichnis", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

Das von dir gewählte Exportverzeichnis existiert nicht.

Bitte wähle einen gültigen Ordner.

", "SUBSCRIPTION_VERIFICATION_ERROR": "Verifizierung des Abonnements fehlgeschlagen", diff --git a/web/packages/base/locales/el-GR/translation.json b/web/packages/base/locales/el-GR/translation.json index 234f9c1874..6e0c20264d 100644 --- a/web/packages/base/locales/el-GR/translation.json +++ b/web/packages/base/locales/el-GR/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "Ενεργοποίηση", + "enable": "Ενεργοποίηση", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "Εσφαλμένος κωδικός", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "Απενεργοποίηση", - "RECONFIGURE": "", + "disable": "Απενεργοποίηση", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "Ενημέρωση", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Ενεργοποίηση αναγνώρισης προσώπου", - "ENABLE_FACE_SEARCH_TITLE": "Ενεργοποίηση αναγνώρισης προσώπου;", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "Ενεργοποίηση αναγνώρισης προσώπου", + "enable_face_search_title": "Ενεργοποίηση αναγνώρισης προσώπου;", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "Εργαστήρια", + "face_search_confirmation": "", + "labs": "Εργαστήρια", "YOURS": "", "passphrase_strength_weak": "Ισχύς κωδικού πρόσβασης: Ασθενής", "passphrase_strength_moderate": "Ισχύς κωδικού πρόσβασης: Μέτριος", "passphrase_strength_strong": "Ισχύς κωδικού πρόσβασης: Ισχυρός", - "PREFERENCES": "Προτιμήσεις", - "LANGUAGE": "Γλώσσα", + "preferences": "Προτιμήσεις", + "language": "Γλώσσα", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Μη έγκυρος κατάλογος εξαγωγής", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "Η επαλήθευση της συνδρομής απέτυχε", diff --git a/web/packages/base/locales/en-US/translation.json b/web/packages/base/locales/en-US/translation.json index b3f4e796c7..eda3f97696 100644 --- a/web/packages/base/locales/en-US/translation.json +++ b/web/packages/base/locales/en-US/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Please enter this code in your favorite authenticator app", "SCAN_QR_CODE": "Scan QR code instead", "ENABLE_TWO_FACTOR": "Enable two-factor", - "ENABLE": "Enable", + "enable": "Enable", "enabled": "Enabled", "LOST_DEVICE": "Lost two-factor device", "INCORRECT_CODE": "Incorrect code", "TWO_FACTOR_INFO": "Add an additional layer of security by requiring more than your email and password to log in to your account", "DISABLE_TWO_FACTOR_LABEL": "Disable two-factor authentication", "UPDATE_TWO_FACTOR_LABEL": "Update your authenticator device", - "DISABLE": "Disable", - "RECONFIGURE": "Reconfigure", + "disable": "Disable", + "reconfigure": "Reconfigure", "UPDATE_TWO_FACTOR": "Update two-factor", "UPDATE_TWO_FACTOR_MESSAGE": "Continuing forward will void any previously configured authenticators", "UPDATE": "Update", @@ -488,18 +488,18 @@ "indexing_status_done": "Done", "ml_search_disable": "Disable face and magic search", "ml_search_disable_confirm": "Do you want to disable face and magic search on all your devices?", - "ENABLE_FACE_SEARCH": "Enable face recognition", - "ENABLE_FACE_SEARCH_TITLE": "Enable face recognition?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

If you enable face recognition, Ente will extract face geometry from your photos. This will happen on your device, and any generated biometric data will be end-to-encrypted.

Please click here for more details about this feature in our privacy policy

", + "enable_face_search": "Enable face recognition", + "enable_face_search_title": "Enable face recognition?", + "enable_face_search_description": "

If you enable face recognition, Ente will extract face geometry from your photos. This will happen on your device, and any generated biometric data will be end-to-encrypted.

Please click here for more details about this feature in our privacy policy

", "ADVANCED": "Advanced", - "FACE_SEARCH_CONFIRMATION": "I understand, and wish to allow Ente to process face geometry", - "LABS": "Labs", + "face_search_confirmation": "I understand, and wish to allow Ente to process face geometry", + "labs": "Labs", "YOURS": "yours", "passphrase_strength_weak": "Password strength: Weak", "passphrase_strength_moderate": "Password strength: Moderate", "passphrase_strength_strong": "Password strength: Strong", - "PREFERENCES": "Preferences", - "LANGUAGE": "Language", + "preferences": "Preferences", + "language": "Language", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Invalid export directory", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

The export directory you have selected does not exist.

Please select a valid directory.

", "SUBSCRIPTION_VERIFICATION_ERROR": "Subscription verification failed", diff --git a/web/packages/base/locales/es-ES/translation.json b/web/packages/base/locales/es-ES/translation.json index cc9d0b8f53..87f0cbe732 100644 --- a/web/packages/base/locales/es-ES/translation.json +++ b/web/packages/base/locales/es-ES/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Por favor, introduce este código en tu aplicación de autenticación favorita", "SCAN_QR_CODE": "Escanear código QR en su lugar", "ENABLE_TWO_FACTOR": "Activar dos factores", - "ENABLE": "Activar", + "enable": "Activar", "enabled": "", "LOST_DEVICE": "Perdido el dispositivo de doble factor", "INCORRECT_CODE": "Código incorrecto", "TWO_FACTOR_INFO": "Añade una capa adicional de seguridad al requerir más de tu email y contraseña para iniciar sesión en tu cuenta", "DISABLE_TWO_FACTOR_LABEL": "Deshabilitar la autenticación de dos factores", "UPDATE_TWO_FACTOR_LABEL": "Actualice su dispositivo de autenticación", - "DISABLE": "Desactivar", - "RECONFIGURE": "Reconfigurar", + "disable": "Desactivar", + "reconfigure": "Reconfigurar", "UPDATE_TWO_FACTOR": "Actualizar doble factor", "UPDATE_TWO_FACTOR_MESSAGE": "Continuar adelante anulará los autenticadores previamente configurados", "UPDATE": "Actualizar", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Activar búsqueda facial", - "ENABLE_FACE_SEARCH_TITLE": "Activar búsqueda facial?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

Si activas la búsqueda facial, ente extraerá la geometría facial de tus fotos. Esto sucederá en su dispositivo y cualquier dato biométrico generado será cifrado de extremo a extremo.

Haga clic aquí para obtener más detalles sobre esta característica en nuestra política de privacidad

", + "enable_face_search": "Activar búsqueda facial", + "enable_face_search_title": "Activar búsqueda facial?", + "enable_face_search_description": "

Si activas la búsqueda facial, ente extraerá la geometría facial de tus fotos. Esto sucederá en su dispositivo y cualquier dato biométrico generado será cifrado de extremo a extremo.

Haga clic aquí para obtener más detalles sobre esta característica en nuestra política de privacidad

", "ADVANCED": "Avanzado", - "FACE_SEARCH_CONFIRMATION": "Comprendo y deseo permitir que ente procese la geometría de la cara", - "LABS": "Labs", + "face_search_confirmation": "Comprendo y deseo permitir que ente procese la geometría de la cara", + "labs": "Labs", "YOURS": "tuyo", "passphrase_strength_weak": "Fortaleza de la contraseña: débil", "passphrase_strength_moderate": "Fortaleza de contraseña: Moderar", "passphrase_strength_strong": "Fortaleza de contraseña: fuerte", - "PREFERENCES": "Preferencias", - "LANGUAGE": "Idioma", + "preferences": "Preferencias", + "language": "Idioma", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Archivo de exportación inválido", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

El directorio de exportación seleccionado no existe.

Por favor, seleccione un directorio válido.

", "SUBSCRIPTION_VERIFICATION_ERROR": "Falló la verificación de la suscripción", diff --git a/web/packages/base/locales/fa-IR/translation.json b/web/packages/base/locales/fa-IR/translation.json index e75d10e3db..9855b2af9f 100644 --- a/web/packages/base/locales/fa-IR/translation.json +++ b/web/packages/base/locales/fa-IR/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/fi-FI/translation.json b/web/packages/base/locales/fi-FI/translation.json index 85d6e9c46e..c4d5ae3a17 100644 --- a/web/packages/base/locales/fi-FI/translation.json +++ b/web/packages/base/locales/fi-FI/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/fr-FR/translation.json b/web/packages/base/locales/fr-FR/translation.json index 9ae20c4d51..5c31a96b31 100644 --- a/web/packages/base/locales/fr-FR/translation.json +++ b/web/packages/base/locales/fr-FR/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Veuillez saisir ce code dans votre appli d'authentification", "SCAN_QR_CODE": "Scannez le QRCode de préférence", "ENABLE_TWO_FACTOR": "Activer la double-authentification", - "ENABLE": "Activer", + "enable": "Activer", "enabled": "", "LOST_DEVICE": "Perte de l'appareil identificateur", "INCORRECT_CODE": "Code non valide", "TWO_FACTOR_INFO": "Rajoutez une couche de sécurité supplémentaire afin de pas utiliser simplement votre e-mail et mot de passe pour vous connecter à votre compte", "DISABLE_TWO_FACTOR_LABEL": "Désactiver la double-authentification", "UPDATE_TWO_FACTOR_LABEL": "Mise à jour de votre appareil identificateur", - "DISABLE": "Désactiver", - "RECONFIGURE": "Reconfigurer", + "disable": "Désactiver", + "reconfigure": "Reconfigurer", "UPDATE_TWO_FACTOR": "Mise à jour de la double-authentification", "UPDATE_TWO_FACTOR_MESSAGE": "Continuer annulera tous les identificateurs précédemment configurés", "UPDATE": "Mise à jour", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Activer la recherche faciale", - "ENABLE_FACE_SEARCH_TITLE": "Activer la recherche faciale ?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

If you enable face search, Ente will extract face geometry from your photos. This will happen on your device, and any generated biometric data will be end-to-encrypted.

Please click here for more details about this feature in our privacy policy

", + "enable_face_search": "Activer la recherche faciale", + "enable_face_search_title": "Activer la recherche faciale ?", + "enable_face_search_description": "

If you enable face search, Ente will extract face geometry from your photos. This will happen on your device, and any generated biometric data will be end-to-encrypted.

Please click here for more details about this feature in our privacy policy

", "ADVANCED": "Avancé", - "FACE_SEARCH_CONFIRMATION": "Je comprends, et je souhaite permettre à Ente de traiter la géométrie faciale", - "LABS": "Labs", + "face_search_confirmation": "Je comprends, et je souhaite permettre à Ente de traiter la géométrie faciale", + "labs": "Labs", "YOURS": "Le vôtre", "passphrase_strength_weak": "Sécurité du mot de passe : faible", "passphrase_strength_moderate": "Sécurité du mot de passe : moyenne", "passphrase_strength_strong": "Sécurité du mot de passe : forte", - "PREFERENCES": "Préférences", - "LANGUAGE": "Langue", + "preferences": "Préférences", + "language": "Langue", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Dossier d'export invalide", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

Le dossier d'export que vous avez sélectionné n'existe pas

Veuillez sélectionner un dossier valide

", "SUBSCRIPTION_VERIFICATION_ERROR": "Échec de la vérification de l'abonnement", diff --git a/web/packages/base/locales/gu-IN/translation.json b/web/packages/base/locales/gu-IN/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/gu-IN/translation.json +++ b/web/packages/base/locales/gu-IN/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/hi-IN/translation.json b/web/packages/base/locales/hi-IN/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/hi-IN/translation.json +++ b/web/packages/base/locales/hi-IN/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/id-ID/translation.json b/web/packages/base/locales/id-ID/translation.json index 2caab67e55..af3be4a5d4 100644 --- a/web/packages/base/locales/id-ID/translation.json +++ b/web/packages/base/locales/id-ID/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Masukkan kode ini ke app autentikator favoritmu", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "Aktifkan autentikasi dua langkah", - "ENABLE": "Aktifkan", + "enable": "Aktifkan", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "Kode salah", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "Nonaktifkan autentikasi dua langkah", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "Nonaktifkan", - "RECONFIGURE": "", + "disable": "Nonaktifkan", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Aktifkan pengenalan wajah", - "ENABLE_FACE_SEARCH_TITLE": "Aktifkan pengenalan wajah?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "Aktifkan pengenalan wajah", + "enable_face_search_title": "Aktifkan pengenalan wajah?", + "enable_face_search_description": "", "ADVANCED": "Lanjutan", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "Keamanan sandi: Lemah", "passphrase_strength_moderate": "Keamanan sandi: Sedang", "passphrase_strength_strong": "Keamanan sandi: Kuat", - "PREFERENCES": "Preferensi", - "LANGUAGE": "Bahasa", + "preferences": "Preferensi", + "language": "Bahasa", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/is-IS/translation.json b/web/packages/base/locales/is-IS/translation.json index 0741125758..e5c22fbbd2 100644 --- a/web/packages/base/locales/is-IS/translation.json +++ b/web/packages/base/locales/is-IS/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/it-IT/translation.json b/web/packages/base/locales/it-IT/translation.json index 1cd8f128b9..2a436d006a 100644 --- a/web/packages/base/locales/it-IT/translation.json +++ b/web/packages/base/locales/it-IT/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Inserisci questo codice nella tua app di autenticazione preferita", "SCAN_QR_CODE": "Oppure scansiona il codice QR", "ENABLE_TWO_FACTOR": "Attiva due fattori", - "ENABLE": "Attiva", + "enable": "Attiva", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "Codice errato", "TWO_FACTOR_INFO": "Aggiungi un ulteriore livello di sicurezza richiedendo più informazioni rispetto a email e password per eseguire l'accesso al tuo account", "DISABLE_TWO_FACTOR_LABEL": "Disabilita l'autenticazione a due fattori", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "Aggiorna", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "Avanzate", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "Sicurezza password: Debole", "passphrase_strength_moderate": "Sicurezza password: Moderata", "passphrase_strength_strong": "Sicurezza password: Forte", - "PREFERENCES": "", - "LANGUAGE": "Lingua", + "preferences": "", + "language": "Lingua", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/ja-JP/translation.json b/web/packages/base/locales/ja-JP/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/ja-JP/translation.json +++ b/web/packages/base/locales/ja-JP/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/ko-KR/translation.json b/web/packages/base/locales/ko-KR/translation.json index 22dc0da971..7c5e47a0d7 100644 --- a/web/packages/base/locales/ko-KR/translation.json +++ b/web/packages/base/locales/ko-KR/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/nl-NL/translation.json b/web/packages/base/locales/nl-NL/translation.json index 2faa3c5bda..6e3cd68a55 100644 --- a/web/packages/base/locales/nl-NL/translation.json +++ b/web/packages/base/locales/nl-NL/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Voer deze code in in uw favoriete verificatie app", "SCAN_QR_CODE": "Scan QR-code in plaats daarvan", "ENABLE_TWO_FACTOR": "Tweestapsverificatie inschakelen", - "ENABLE": "Inschakelen", + "enable": "Inschakelen", "enabled": "", "LOST_DEVICE": "Tweestapsverificatie apparaat verloren", "INCORRECT_CODE": "Onjuiste code", "TWO_FACTOR_INFO": "Voeg een extra beveiligingslaag toe door meer dan uw e-mailadres en wachtwoord te vereisen om in te loggen op uw account", "DISABLE_TWO_FACTOR_LABEL": "Schakel tweestapsverificatie uit", "UPDATE_TWO_FACTOR_LABEL": "Update uw verificatie apparaat", - "DISABLE": "Uitschakelen", - "RECONFIGURE": "Herconfigureren", + "disable": "Uitschakelen", + "reconfigure": "Herconfigureren", "UPDATE_TWO_FACTOR": "Tweestapsverificatie bijwerken", "UPDATE_TWO_FACTOR_MESSAGE": "Verder gaan zal elk eerder geconfigureerde verificatie apparaat ontzeggen", "UPDATE": "Bijwerken", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Zoeken op gezichten inschakelen", - "ENABLE_FACE_SEARCH_TITLE": "Zoeken op gezichten inschakelen?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

Als u zoeken op gezichten inschakelt, analyseert Ente de gezichtsgeometrie uit uw foto's. Dit gebeurt op uw apparaat en alle gegenereerde biometrische gegevens worden end-to-end versleuteld.

Klik hier voor meer informatie over deze functie in ons privacybeleid

", + "enable_face_search": "Zoeken op gezichten inschakelen", + "enable_face_search_title": "Zoeken op gezichten inschakelen?", + "enable_face_search_description": "

Als u zoeken op gezichten inschakelt, analyseert Ente de gezichtsgeometrie uit uw foto's. Dit gebeurt op uw apparaat en alle gegenereerde biometrische gegevens worden end-to-end versleuteld.

Klik hier voor meer informatie over deze functie in ons privacybeleid

", "ADVANCED": "Geavanceerd", - "FACE_SEARCH_CONFIRMATION": "Ik begrijp het, en wil Ente toestaan om gezichten te analyseren", - "LABS": "Lab's", + "face_search_confirmation": "Ik begrijp het, en wil Ente toestaan om gezichten te analyseren", + "labs": "Lab's", "YOURS": "jouw", "passphrase_strength_weak": "Wachtwoord sterkte: Zwak", "passphrase_strength_moderate": "Wachtwoord sterkte: Matig", "passphrase_strength_strong": "Wachtwoord sterkte: Sterk", - "PREFERENCES": "Instellingen", - "LANGUAGE": "Taal", + "preferences": "Instellingen", + "language": "Taal", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Ongeldige export map", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

De export map die u heeft geselecteerd bestaat niet.

Selecteer een geldige map.

", "SUBSCRIPTION_VERIFICATION_ERROR": "Abonnementsverificatie mislukt", diff --git a/web/packages/base/locales/pl-PL/translation.json b/web/packages/base/locales/pl-PL/translation.json index 02d9d8b8dc..87948ac3b4 100644 --- a/web/packages/base/locales/pl-PL/translation.json +++ b/web/packages/base/locales/pl-PL/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Wprowadź ten kod w swojej ulubionej aplikacji uwierzytelniającej", "SCAN_QR_CODE": "Zamiast tego zeskanuj kod QR", "ENABLE_TWO_FACTOR": "Włącz uwierzytelnianie dwustopniowe", - "ENABLE": "Włącz", + "enable": "Włącz", "enabled": "", "LOST_DEVICE": "Utracono urządzenie dwustopniowe", "INCORRECT_CODE": "Nieprawidłowy kod", "TWO_FACTOR_INFO": "Dodaj dodatkową warstwę bezpieczeństwa, wymagając więcej niż Twojego adresu e-mail i hasła, aby zalogować się na swoje konto", "DISABLE_TWO_FACTOR_LABEL": "Wyłącz uwierzytelnianie dwustopniowe", "UPDATE_TWO_FACTOR_LABEL": "Zaktualizuj swoje urządzenie uwierzytelniające", - "DISABLE": "Wyłącz", - "RECONFIGURE": "Konfiguruj ponownie", + "disable": "Wyłącz", + "reconfigure": "Konfiguruj ponownie", "UPDATE_TWO_FACTOR": "Aktualizuj uwierzytelnianie dwustopniowe", "UPDATE_TWO_FACTOR_MESSAGE": "Kontynuowanie spowoduje unieważnienie wszystkich poprzednio skonfigurowanych uwierzytelniaczy", "UPDATE": "Aktualizuj", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Włącz rozpoznawanie twarzy", - "ENABLE_FACE_SEARCH_TITLE": "Włączyć rozpoznawanie twarzy?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

Jeśli włączysz rozpoznawanie twarzy, Ente wyodrębni geometrię twarzy ze zdjęć. Będzie to miało miejsce na Twoim urządzeniu, a wszystkie wygenerowane dane biometryczne będą zaszyfrowane metodą end-to-end.

Kliknij tutaj, aby uzyskać więcej informacji na temat tej funkcji w naszej polityce prywatności

", + "enable_face_search": "Włącz rozpoznawanie twarzy", + "enable_face_search_title": "Włączyć rozpoznawanie twarzy?", + "enable_face_search_description": "

Jeśli włączysz rozpoznawanie twarzy, Ente wyodrębni geometrię twarzy ze zdjęć. Będzie to miało miejsce na Twoim urządzeniu, a wszystkie wygenerowane dane biometryczne będą zaszyfrowane metodą end-to-end.

Kliknij tutaj, aby uzyskać więcej informacji na temat tej funkcji w naszej polityce prywatności

", "ADVANCED": "Zaawansowane", - "FACE_SEARCH_CONFIRMATION": "Rozumiem i chcę pozwolić Ente na przetwarzanie geometrii twarzy", - "LABS": "Laboratoria", + "face_search_confirmation": "Rozumiem i chcę pozwolić Ente na przetwarzanie geometrii twarzy", + "labs": "Laboratoria", "YOURS": "twoje", "passphrase_strength_weak": "Siła hasła: Słabe", "passphrase_strength_moderate": "Siła hasła: Umiarkowane", "passphrase_strength_strong": "Siła hasła: Silne", - "PREFERENCES": "Preferencje", - "LANGUAGE": "Język", + "preferences": "Preferencje", + "language": "Język", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Nieprawidłowy katalog eksportu", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

Wybrany katalog eksportu nie istnieje.

Proszę wybrać prawidłowy katalog.

", "SUBSCRIPTION_VERIFICATION_ERROR": "Weryfikacja subskrypcji nie powiodła się", diff --git a/web/packages/base/locales/pt-BR/translation.json b/web/packages/base/locales/pt-BR/translation.json index 3b9ff6156d..3bbcbdf7dc 100644 --- a/web/packages/base/locales/pt-BR/translation.json +++ b/web/packages/base/locales/pt-BR/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Por favor, insira este código no seu aplicativo autenticador favorito", "SCAN_QR_CODE": "Em vez disso, escaneie um Código QR", "ENABLE_TWO_FACTOR": "Ativar autenticação de dois fatores", - "ENABLE": "Habilitar", + "enable": "Habilitar", "enabled": "", "LOST_DEVICE": "Dispositivo de dois fatores perdido", "INCORRECT_CODE": "Código incorreto", "TWO_FACTOR_INFO": "Adicione uma camada adicional de segurança, exigindo mais do que seu e-mail e senha para entrar na sua conta", "DISABLE_TWO_FACTOR_LABEL": "Desativar autenticação de dois fatores", "UPDATE_TWO_FACTOR_LABEL": "Atualize seu dispositivo autenticador", - "DISABLE": "Desativar", - "RECONFIGURE": "Reconfigurar", + "disable": "Desativar", + "reconfigure": "Reconfigurar", "UPDATE_TWO_FACTOR": "Atualizar dois fatores", "UPDATE_TWO_FACTOR_MESSAGE": "Continuar adiante anulará qualquer autenticador configurado anteriormente", "UPDATE": "Atualização", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Habilitar reconhecimento facial", - "ENABLE_FACE_SEARCH_TITLE": "Habilitar reconhecimento facial?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

Se você habilitar o reconhecimento facial, o aplicativo extrairá a geometria do rosto de suas fotos. Isso ocorrerá em seu dispositivo, e quaisquer dados biométricos gerados serão criptografados de ponta a ponta.

Por favor, clique aqui para obter mais detalhes sobre esta funcionalidade em nossa política de privacidade

", + "enable_face_search": "Habilitar reconhecimento facial", + "enable_face_search_title": "Habilitar reconhecimento facial?", + "enable_face_search_description": "

Se você habilitar o reconhecimento facial, o aplicativo extrairá a geometria do rosto de suas fotos. Isso ocorrerá em seu dispositivo, e quaisquer dados biométricos gerados serão criptografados de ponta a ponta.

Por favor, clique aqui para obter mais detalhes sobre esta funcionalidade em nossa política de privacidade

", "ADVANCED": "Avançado", - "FACE_SEARCH_CONFIRMATION": "Eu entendo, e desejo permitir que o ente processe a geometria do rosto", - "LABS": "Laboratórios", + "face_search_confirmation": "Eu entendo, e desejo permitir que o ente processe a geometria do rosto", + "labs": "Laboratórios", "YOURS": "seu", "passphrase_strength_weak": "Força da senha: fraca", "passphrase_strength_moderate": "Força da senha: moderada", "passphrase_strength_strong": "Força da senha: forte", - "PREFERENCES": "Preferências", - "LANGUAGE": "Idioma", + "preferences": "Preferências", + "language": "Idioma", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Diretório de exportação inválido", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

O diretório de exportação que você selecionou não existe.

Por favor, selecione um diretório válido.

", "SUBSCRIPTION_VERIFICATION_ERROR": "Falha na verificação de assinatura", diff --git a/web/packages/base/locales/pt-PT/translation.json b/web/packages/base/locales/pt-PT/translation.json index 811552fe21..92be64e00a 100644 --- a/web/packages/base/locales/pt-PT/translation.json +++ b/web/packages/base/locales/pt-PT/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/ru-RU/translation.json b/web/packages/base/locales/ru-RU/translation.json index 7018e63b61..06fe59e6c3 100644 --- a/web/packages/base/locales/ru-RU/translation.json +++ b/web/packages/base/locales/ru-RU/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "Пожалуйста, введите этот код в вашем любимом приложении для аутентификации", "SCAN_QR_CODE": "Сканировать QR-код вместо", "ENABLE_TWO_FACTOR": "Включить двухфакторную аутентификацию", - "ENABLE": "Включить", + "enable": "Включить", "enabled": "", "LOST_DEVICE": "Потеряно двухфакторное устройство", "INCORRECT_CODE": "Неверный код", "TWO_FACTOR_INFO": "Добавьте дополнительный уровень безопасности, запросив для входа в свою учетную запись не только адрес электронной почты и пароль", "DISABLE_TWO_FACTOR_LABEL": "Отключить двухфакторную аутентификацию", "UPDATE_TWO_FACTOR_LABEL": "Обновите свое устройство аутентификации", - "DISABLE": "Отключить", - "RECONFIGURE": "Перенастроить", + "disable": "Отключить", + "reconfigure": "Перенастроить", "UPDATE_TWO_FACTOR": "Обновить двухфакторную аутентификацию", "UPDATE_TWO_FACTOR_MESSAGE": "Дальнейшая переадресация приведет к аннулированию всех ранее настроенных средств проверки подлинности", "UPDATE": "Обновить", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "Включить распознавание лиц", - "ENABLE_FACE_SEARCH_TITLE": "Включить распознавание лиц?", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

Если вы включите функцию распознавания лиц, Ente извлечет геометрию лица из ваших фотографий. Это произойдет на вашем устройстве, и все сгенерированные биометрические данные будут зашифрованы полностью.

Пожалуйста, нажмите здесь для получения более подробной информации об этой функции в нашей политике конфиденциальности

", + "enable_face_search": "Включить распознавание лиц", + "enable_face_search_title": "Включить распознавание лиц?", + "enable_face_search_description": "

Если вы включите функцию распознавания лиц, Ente извлечет геометрию лица из ваших фотографий. Это произойдет на вашем устройстве, и все сгенерированные биометрические данные будут зашифрованы полностью.

Пожалуйста, нажмите здесь для получения более подробной информации об этой функции в нашей политике конфиденциальности

", "ADVANCED": "Передовой", - "FACE_SEARCH_CONFIRMATION": "Я понимаю и хочу позволить Ente обрабатывать геометрию грани", - "LABS": "Лаборатории", + "face_search_confirmation": "Я понимаю и хочу позволить Ente обрабатывать геометрию грани", + "labs": "Лаборатории", "YOURS": "твой", "passphrase_strength_weak": "Надежность пароля: слабая", "passphrase_strength_moderate": "Надежность пароля: Умеренная", "passphrase_strength_strong": "Надежность пароля: Надежный", - "PREFERENCES": "Предпочтения", - "LANGUAGE": "Язык", + "preferences": "Предпочтения", + "language": "Язык", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "Недопустимый каталог экспорта", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

Выбранный вами каталог экспорта не существует.

Пожалуйста, выберите правильный каталог.

", "SUBSCRIPTION_VERIFICATION_ERROR": "Не удалось подтвердить подписку", diff --git a/web/packages/base/locales/sv-SE/translation.json b/web/packages/base/locales/sv-SE/translation.json index 45ad964f89..0ed1825aa7 100644 --- a/web/packages/base/locales/sv-SE/translation.json +++ b/web/packages/base/locales/sv-SE/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "Aktivera", + "enable": "Aktivera", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "Felaktig kod", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "Inaktivera tvåfaktorsautentisering", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "Inaktivera", - "RECONFIGURE": "", + "disable": "Inaktivera", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "Uppdatera", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "Språk", + "preferences": "", + "language": "Språk", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/te-IN/translation.json b/web/packages/base/locales/te-IN/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/te-IN/translation.json +++ b/web/packages/base/locales/te-IN/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/th-TH/translation.json b/web/packages/base/locales/th-TH/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/th-TH/translation.json +++ b/web/packages/base/locales/th-TH/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/ti-ER/translation.json b/web/packages/base/locales/ti-ER/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/ti-ER/translation.json +++ b/web/packages/base/locales/ti-ER/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/tr-TR/translation.json b/web/packages/base/locales/tr-TR/translation.json index 073ff7b391..938b0a3d75 100644 --- a/web/packages/base/locales/tr-TR/translation.json +++ b/web/packages/base/locales/tr-TR/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "", "SCAN_QR_CODE": "", "ENABLE_TWO_FACTOR": "", - "ENABLE": "", + "enable": "", "enabled": "", "LOST_DEVICE": "", "INCORRECT_CODE": "", "TWO_FACTOR_INFO": "", "DISABLE_TWO_FACTOR_LABEL": "", "UPDATE_TWO_FACTOR_LABEL": "", - "DISABLE": "", - "RECONFIGURE": "", + "disable": "", + "reconfigure": "", "UPDATE_TWO_FACTOR": "", "UPDATE_TWO_FACTOR_MESSAGE": "", "UPDATE": "", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "", - "ENABLE_FACE_SEARCH_TITLE": "", - "ENABLE_FACE_SEARCH_DESCRIPTION": "", + "enable_face_search": "", + "enable_face_search_title": "", + "enable_face_search_description": "", "ADVANCED": "", - "FACE_SEARCH_CONFIRMATION": "", - "LABS": "", + "face_search_confirmation": "", + "labs": "", "YOURS": "", "passphrase_strength_weak": "", "passphrase_strength_moderate": "", "passphrase_strength_strong": "", - "PREFERENCES": "", - "LANGUAGE": "", + "preferences": "", + "language": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "", "SUBSCRIPTION_VERIFICATION_ERROR": "", diff --git a/web/packages/base/locales/zh-CN/translation.json b/web/packages/base/locales/zh-CN/translation.json index e0faf85e7e..9b5a29f387 100644 --- a/web/packages/base/locales/zh-CN/translation.json +++ b/web/packages/base/locales/zh-CN/translation.json @@ -263,15 +263,15 @@ "TWO_FACTOR_MANUAL_CODE_INSTRUCTION": "请在您最喜欢的验证器应用中输入此代码", "SCAN_QR_CODE": "改为扫描二维码", "ENABLE_TWO_FACTOR": "启用双重认证", - "ENABLE": "启用", + "enable": "启用", "enabled": "", "LOST_DEVICE": "丢失了双重认证设备", "INCORRECT_CODE": "代码错误", "TWO_FACTOR_INFO": "登录账户时需要的不仅仅是电子邮件和密码，这增加了额外的安全层", "DISABLE_TWO_FACTOR_LABEL": "禁用双重认证", "UPDATE_TWO_FACTOR_LABEL": "更新您的身份验证器设备", - "DISABLE": "禁用", - "RECONFIGURE": "重新配置", + "disable": "禁用", + "reconfigure": "重新配置", "UPDATE_TWO_FACTOR": "更新双重认证", "UPDATE_TWO_FACTOR_MESSAGE": "向前继续将使之前配置的任何身份验证器失效", "UPDATE": "更新", @@ -488,18 +488,18 @@ "indexing_status_done": "", "ml_search_disable": "", "ml_search_disable_confirm": "", - "ENABLE_FACE_SEARCH": "启用面部搜索", - "ENABLE_FACE_SEARCH_TITLE": "要启用面部搜索吗？", - "ENABLE_FACE_SEARCH_DESCRIPTION": "

如果您启用面部搜索，Ente 将从照片中提取脸部几何形状。 这将发生在您的设备上，任何生成的生物测定数据都将是端到端加密的。

请单击此处以在我们的隐私政策中了解有关此功能的更多详细信息

", + "enable_face_search": "启用面部搜索", + "enable_face_search_title": "要启用面部搜索吗？", + "enable_face_search_description": "

如果您启用面部搜索，Ente 将从照片中提取脸部几何形状。 这将发生在您的设备上，任何生成的生物测定数据都将是端到端加密的。

请单击此处以在我们的隐私政策中了解有关此功能的更多详细信息

", "ADVANCED": "高级设置", - "FACE_SEARCH_CONFIRMATION": "我理解，并希望允许Ente处理面部几何形状", - "LABS": "实验室", + "face_search_confirmation": "我理解，并希望允许Ente处理面部几何形状", + "labs": "实验室", "YOURS": "你的", "passphrase_strength_weak": "密码强度：较弱", "passphrase_strength_moderate": "密码强度：中度", "passphrase_strength_strong": "密码强度：强", - "PREFERENCES": "首选项", - "LANGUAGE": "语言", + "preferences": "首选项", + "language": "语言", "EXPORT_DIRECTORY_DOES_NOT_EXIST": "无效的导出目录", "EXPORT_DIRECTORY_DOES_NOT_EXIST_MESSAGE": "

您选择的导出目录不存在。

请选择一个有效的目录。

", "SUBSCRIPTION_VERIFICATION_ERROR": "订阅验证失败", diff --git a/web/packages/new/photos/components/MLSettings.tsx b/web/packages/new/photos/components/MLSettings.tsx index a3d12a41cc..ad31fa04b1 100644 --- a/web/packages/new/photos/components/MLSettings.tsx +++ b/web/packages/new/photos/components/MLSettings.tsx @@ -164,7 +164,7 @@ const EnableML: React.FC = ({ onEnable }) => {