Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@etnoy
etnoy committed Jan 11, 2025
1 parent 6be2662 commit 3465bfd
Show file tree
Hide file tree
Showing 6 changed files with 111 additions and 35 deletions.
6 changes: 3 additions & 3 deletions e2e/src/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -454,12 +454,12 @@ export const utils = {
createLibrary: (accessToken: string, dto: CreateLibraryDto) =>
createLibrary({ createLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),

updateLibrary: (accessToken: string, id: string, dto: UpdateLibraryDto) =>
updateLibrary({ id, updateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),

validateLibrary: (accessToken: string, id: string, dto: ValidateLibraryDto) =>
validate({ id, validateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),

updateLibrary: (accessToken: string, id: string, dto: UpdateLibraryDto) =>
updateLibrary({ id, updateLibraryDto: dto }, { headers: asBearerAuth(accessToken) }),

createPartner: (accessToken: string, id: string) => createPartner({ id }, { headers: asBearerAuth(accessToken) }),

updateMyPreferences: (accessToken: string, userPreferencesUpdateDto: UserPreferencesUpdateDto) =>
Expand Down
6 changes: 5 additions & 1 deletion server/src/interfaces/job.interface.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ export interface IDelayedJob extends IBaseJob {

export interface IEntityJob extends IBaseJob {
id: string;
source?: 'upload' | 'library-import' | 'sidecar-write' | 'copy';
source?: 'upload' | 'sidecar-write' | 'copy';
notify?: boolean;
}

Expand All @@ -147,11 +147,15 @@ export interface ILibraryFileJob {
libraryId: string;
ownerId: string;
assetPaths: string[];
progressCounter?: number;
totalAssets?: number;
}

export interface ILibraryBulkIdsJob {
libraryId: string;
assetIds: string[];
progressCounter?: number;
totalAssets?: number;
}

export interface IBulkEntityJob {
Expand Down
1 change: 1 addition & 0 deletions server/src/interfaces/library.interface.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ export enum AssetSyncResult {
DO_NOTHING,
UPDATE,
OFFLINE,
ONLINE,
}

export interface ILibraryRepository {
Expand Down
2 changes: 1 addition & 1 deletion server/src/services/job.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,7 +266,7 @@ export class JobService extends BaseService {
}

case JobName.GENERATE_THUMBNAILS: {
if (!item.data.notify && item.data.source !== 'upload' && item.data.source !== 'library-import') {
if (!item.data.notify && item.data.source !== 'upload') {
break;
}

Expand Down
125 changes: 100 additions & 25 deletions server/src/services/library.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,15 @@ export class LibraryService extends BaseService {
assetIds.push(...batchIds);
}

this.logger.log(`Imported ${assetIds.length} file(s) into library ${job.libraryId}`);
let progressMessage = '';

if (job.progressCounter && job.totalAssets) {
progressMessage = `(${job.progressCounter} of ${job.totalAssets}) `;
} else {
progressMessage = `(${job.progressCounter} done so far) `;
}

this.logger.log(`Imported ${assetIds.length} ${progressMessage}file(s) into library ${job.libraryId}`);

await this.queuePostSyncJobs(assetIds);

Expand Down Expand Up @@ -380,8 +388,7 @@ export class LibraryService extends BaseService {
await this.jobRepository.queueAll(
assetIds.map((assetId) => ({
name: JobName.SIDECAR_DISCOVERY,
data: { id: assetId },
source: 'upload',
data: { id: assetId, source: 'upload' },
})),
);
}
Expand Down Expand Up @@ -434,6 +441,7 @@ export class LibraryService extends BaseService {
const assets = await this.assetRepository.getByIds(job.assetIds);

const assetIdsToOffline: string[] = [];
const assetIdsToOnline: string[] = [];
const assetIdsToUpdate: string[] = [];

this.logger.debug(`Checking batch of ${assets.length} existing asset(s) in library ${job.libraryId}`);
Expand All @@ -445,35 +453,79 @@ export class LibraryService extends BaseService {
assetIdsToOffline.push(asset.id);
break;
}
case AssetSyncResult.ONLINE: {
assetIdsToOffline.push(asset.id);
break;
}
case AssetSyncResult.UPDATE: {
assetIdsToUpdate.push(asset.id);
break;
}
}
}

let progressMessage = '';

if (assetIdsToOffline.length > 0) {
await this.assetRepository.updateAll(assetIdsToOffline, {
isOffline: true,
status: AssetStatus.TRASHED,
deletedAt: new Date(),
});

if (progressMessage) {
progressMessage + ', ';
}
progressMessage += `${assetIdsToOffline.length} offlined`;
}

if (assetIdsToUpdate.length > 0) {
if (assetIdsToOnline.length > 0) {
//TODO: When we have asset status, we need to leave deletedAt as is when status is trashed
await this.assetRepository.updateAll(assetIdsToUpdate, {
await this.assetRepository.updateAll(assetIdsToOnline, {
isOffline: false,
status: AssetStatus.ACTIVE,
deletedAt: null,
});
await this.queuePostSyncJobs(assetIdsToOnline);

if (progressMessage) {
progressMessage + ', ';
}

progressMessage += `${assetIdsToOnline.length} onlined`;
}

if (assetIdsToUpdate.length > 0) {
//TODO: When we have asset status, we need to leave deletedAt as is when status is trashed
await this.queuePostSyncJobs(assetIdsToUpdate);

if (progressMessage) {
progressMessage + ', ';
}

progressMessage += `${assetIdsToUpdate.length} updated`;
}

const remainingCount = assets.length - assetIdsToOffline.length - assetIdsToUpdate.length;
const remainingCount = assets.length - assetIdsToOffline.length - assetIdsToUpdate.length - assetIdsToOnline.length;

if (remainingCount) {
if (progressMessage) {
progressMessage + ', ';
}

progressMessage += `${remainingCount} unchanged`;
}

let cumulativeProgressMessage = '';

if (job.progressCounter && job.totalAssets) {
const cumulativePercentage = ((100 * job.progressCounter) / job.totalAssets).toFixed(1);

cumulativeProgressMessage = `(Total progress: ${job.progressCounter} of ${job.totalAssets}, ${cumulativePercentage} %) `;
}

this.logger.log(
`Checked existing asset(s): ${assetIdsToOffline.length} offlined, ${assetIdsToUpdate.length} updated, ${remainingCount} unchanged of batch of ${assets.length} in library ${job.libraryId}.`,
`Checked existing asset(s): ${progressMessage} of current batch of ${assets.length} ${cumulativeProgressMessage}in library ${job.libraryId}.`,
);

return JobStatus.SUCCESS;
Expand Down Expand Up @@ -522,16 +574,31 @@ export class LibraryService extends BaseService {
}

const mtime = stat.mtime;
const isAssetModified = mtime.toISOString() !== asset.fileModifiedAt.toISOString();
const isAssetTimeUpdated = mtime.toISOString() !== asset.fileModifiedAt.toISOString();

let shouldAssetGoOnline = false;

if (asset.isOffline && asset.status != AssetStatus.DELETED) {
// Only perform the expensive check if the asset is offline

// TODO: give more feedback on why asset was onlined
shouldAssetGoOnline = await this.checkOfflineAsset(asset);

if (shouldAssetGoOnline) {
this.logger.debug(`Asset is back online: ${asset.originalPath}`);

return AssetSyncResult.ONLINE;
} else {
this.logger.debug(`Asset is still offline: ${asset.originalPath}`);

return AssetSyncResult.DO_NOTHING;
}
}

if (shouldAssetGoOnline || isAssetModified) {
this.logger.debug(`Asset was offline or modified, updating asset record ${asset.originalPath}`);
if (isAssetTimeUpdated) {
this.logger.verbose(
`Asset ${asset.originalPath} modification time changed from ${asset.fileModifiedAt.toISOString()} to ${mtime.toISOString()}, queuing metadata extraction`,
);

return AssetSyncResult.UPDATE;
}
Expand Down Expand Up @@ -566,6 +633,8 @@ export class LibraryService extends BaseService {
return JobStatus.SKIPPED;
}

let assetsOnDiskCount = 0;

const pathsOnDisk = this.storageRepository.walk({
pathsToCrawl: validImportPaths,
includeHidden: false,
Expand All @@ -576,27 +645,31 @@ export class LibraryService extends BaseService {
let importCount = 0;
let crawlCount = 0;

this.logger.log(`Starting crawl of ${validImportPaths.length} path(s) for library ${library.id}...`);
this.logger.log(`Starting disk crawl of ${validImportPaths.length} import path(s) for library ${library.id}...`);

for await (const pathBatch of pathsOnDisk) {
crawlCount += pathBatch.length;
this.logger.debug(
`Crawled ${pathBatch.length} file(s) for library ${library.id}, in total ${crawlCount} file(s) crawled so far`,
);
const newPaths = await this.assetRepository.getNewPaths(library.id, pathBatch);

if (newPaths.length > 0) {
importCount += newPaths.length;

await this.jobRepository.queue({
name: JobName.LIBRARY_SYNC_FILES,
data: { libraryId: library.id, ownerId: library.ownerId, assetPaths: newPaths },
data: {
libraryId: library.id,
ownerId: library.ownerId,
assetPaths: newPaths,
progressCounter: crawlCount,
totalAssets: assetsOnDiskCount,
},
});
this.logger.log(
`Crawled ${crawlCount} file(s) so far: ${newPaths.length} of current batch queued for import for ${library.id}...`,
`Crawled ${crawlCount} file(s) so far: ${newPaths.length} of current batch of ${pathBatch.length} will be imported to library ${library.id}...`,
);
} else {
this.logger.log(
`Crawled ${crawlCount} file(s) so far: ${pathBatch.length} of current batch already in library ${library.id}...`,
`Crawled ${crawlCount} file(s) so far: All ${pathBatch.length} of current batch already in library ${library.id}...`,
);
}
}
Expand Down Expand Up @@ -633,7 +706,7 @@ export class LibraryService extends BaseService {
}

this.logger.log(
`${assetCount} asset(s) in library ${library.id} will be checked against import paths and exclusion patterns...`,
`Checking ${assetCount} asset(s) against import paths and exclusion patterns in library ${library.id}...`,
);

const offlineResult = await this.assetRepository.updateOffline(library);
Expand All @@ -646,17 +719,15 @@ export class LibraryService extends BaseService {

if (affectedAssetCount === assetCount) {
this.logger.log(
`All ${assetCount} asset(s) in ${library.id} are outside of import paths and/or match an exclusion pattern, marked as offline`,
`All ${assetCount} asset(s) were offlined due to import paths and/or exclusion pattern(s) in ${library.id}`,
);

return JobStatus.SUCCESS;
} else if (affectedAssetCount !== assetCount && affectedAssetCount > 0) {
this.logger.log(
`${offlineResult.affected} asset(s) out of ${assetCount} were marked offline due to import paths and/or exclusion patterns for library ${library.id}`,
);
} else if (affectedAssetCount == 0) {
this.logger.log(`No assets were offlined due to import paths and/or exclusion pattern(s) in ${library.id} `);
} else {
this.logger.log(
`All ${assetCount} asset(s) in library ${library.id} were in an import path and none matched an exclusion pattern`,
`${offlineResult.affected} asset(s) out of ${assetCount} were offlined due to import paths and/or exclusion pattern(s) in library ${library.id}`,
);
}

Expand All @@ -675,11 +746,15 @@ export class LibraryService extends BaseService {
data: {
libraryId: library.id,
assetIds: assets.map((asset) => asset.id),
progressCounter: currentAssetCount,
totalAssets: assetCount,
},
});

const completePercentage = ((100 * currentAssetCount) / assetCount).toFixed(1);

this.logger.log(
`Queued check of ${currentAssetCount} of ${assetCount} existing asset(s) so far in library ${library.id}`,
`Queued check of ${currentAssetCount} of ${assetCount} (${completePercentage} %) existing asset(s) so far in library ${library.id}`,
);
}

Expand Down
6 changes: 1 addition & 5 deletions server/src/services/metadata.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,17 +148,13 @@ export class MetadataService extends BaseService {
}

@OnJob({ name: JobName.METADATA_EXTRACTION, queue: QueueName.METADATA_EXTRACTION })
async handleMetadataExtraction({ id, source }: JobOf<JobName.METADATA_EXTRACTION>): Promise<JobStatus> {
async handleMetadataExtraction({ id }: JobOf<JobName.METADATA_EXTRACTION>): Promise<JobStatus> {
const { metadata, reverseGeocoding } = await this.getConfig({ withCache: true });
const [asset] = await this.assetRepository.getByIds([id], { faces: { person: false } });
if (!asset) {
return JobStatus.FAILED;
}

if (source === 'library-import') {
await this.processSidecar(id, false);
}

const stats = await this.storageRepository.stat(asset.originalPath);

const exifTags = await this.getExifTags(asset);
Expand Down

0 comments on commit 3465bfd

Please sign in to comment.