From a187cf6f714a049ebaeac01488f50379599c5c06 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sun, 15 Oct 2023 15:00:28 -0400 Subject: [PATCH 01/37] Allow in-memory Loki and extent store --- package-lock.json | 33 +++ package.json | 4 +- src/blob/BlobServer.ts | 13 +- src/blob/SqlBlobServer.ts | 6 +- src/blob/persistence/LokiBlobMetadataStore.ts | 7 +- src/common/ConfigurationBase.ts | 1 + .../persistence/LokiExtentMetadataStore.ts | 7 +- src/common/persistence/MemoryExtentStore.ts | 190 ++++++++++++++++++ src/queue/QueueServer.ts | 13 +- .../persistence/LokiQueueMetadataStore.ts | 7 +- src/table/TableServer.ts | 3 +- .../persistence/LokiTableMetadataStore.ts | 7 +- 12 files changed, 272 insertions(+), 19 deletions(-) create mode 100644 src/common/persistence/MemoryExtentStore.ts diff --git a/package-lock.json b/package-lock.json index edcbe3e15..3384478f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "glob-to-regexp": "^0.4.1", "jsonwebtoken": "^9.0.0", "lokijs": "^1.5.6", + "memorystream": "^0.3.1", "morgan": "^1.9.1", "multistream": "^2.1.1", "mysql2": "^3.2.0", @@ -53,6 +54,7 @@ "@types/glob-to-regexp": "^0.4.1", "@types/jsonwebtoken": "^9.0.1", "@types/lokijs": "^1.5.2", + "@types/memorystream": "^0.3.2", "@types/mocha": "^9.0.0", "@types/morgan": "^1.7.35", "@types/multistream": "^2.1.1", @@ -1454,6 +1456,15 @@ "integrity": "sha512-Q/F6OUCZPHWY4hzEowhCswi9Tafc/E7DCUyyWIOH3+hM3K96Mkj2U3byfzs7Yd542I8gT/8oUALnoddqdA20xg==", "dev": true }, + "node_modules/@types/memorystream": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@types/memorystream/-/memorystream-0.3.2.tgz", + "integrity": "sha512-92v1riJ4LA00mKsNf9Ac8ubeOBB7zEO+L+ZCOrd22QSsEUVEy2be2GlZLBt7NKJQBguGBSVhZSuaKf/zATVClw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/mime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", @@ -7189,6 +7200,14 @@ "node": ">= 0.6" } }, + "node_modules/memorystream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", + "integrity": "sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==", + "engines": { + "node": ">= 0.10.0" + } + }, "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -11422,6 +11441,15 @@ "integrity": "sha512-Q/F6OUCZPHWY4hzEowhCswi9Tafc/E7DCUyyWIOH3+hM3K96Mkj2U3byfzs7Yd542I8gT/8oUALnoddqdA20xg==", "dev": true }, + "@types/memorystream": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/@types/memorystream/-/memorystream-0.3.2.tgz", + "integrity": "sha512-92v1riJ4LA00mKsNf9Ac8ubeOBB7zEO+L+ZCOrd22QSsEUVEy2be2GlZLBt7NKJQBguGBSVhZSuaKf/zATVClw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/mime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", @@ -15853,6 +15881,11 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, + "memorystream": { + "version": "0.3.1", + "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", + "integrity": "sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==" + }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", diff --git a/package.json b/package.json index 952136c2f..d9d7642b0 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "glob-to-regexp": "^0.4.1", "jsonwebtoken": "^9.0.0", "lokijs": "^1.5.6", + "memorystream": "^0.3.1", "morgan": "^1.9.1", "multistream": "^2.1.1", "mysql2": "^3.2.0", @@ -58,6 +59,7 @@ "@types/glob-to-regexp": "^0.4.1", "@types/jsonwebtoken": "^9.0.1", "@types/lokijs": "^1.5.2", + "@types/memorystream": "^0.3.2", "@types/mocha": "^9.0.0", "@types/morgan": "^1.7.35", "@types/multistream": "^2.1.1", @@ -320,4 +322,4 @@ "url": "https://github.com/azure/azurite/issues" }, "homepage": "https://github.com/azure/azurite#readme" -} \ No newline at end of file +} diff --git a/src/blob/BlobServer.ts b/src/blob/BlobServer.ts index a70de8350..9c8d3439d 100644 --- a/src/blob/BlobServer.ts +++ b/src/blob/BlobServer.ts @@ -9,6 +9,7 @@ import IGCManager from "../common/IGCManager"; import IRequestListenerFactory from "../common/IRequestListenerFactory"; import logger from "../common/Logger"; import FSExtentStore from "../common/persistence/FSExtentStore"; +import MemoryExtentStore from "../common/persistence/MemoryExtentStore"; import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; import IExtentStore from "../common/persistence/IExtentStore"; import LokiExtentMetadataStore from "../common/persistence/LokiExtentMetadataStore"; @@ -73,15 +74,19 @@ export default class BlobServer extends ServerBase implements ICleaner { // creating a new XXXDataStore class implementing IBlobMetadataStore interface // and replace the default LokiBlobMetadataStore const metadataStore: IBlobMetadataStore = new LokiBlobMetadataStore( - configuration.metadataDBPath - // logger + configuration.metadataDBPath, + configuration.isMemoryPersistence ); const extentMetadataStore: IExtentMetadataStore = new LokiExtentMetadataStore( - configuration.extentDBPath + configuration.extentDBPath, + configuration.isMemoryPersistence ); - const extentStore: IExtentStore = new FSExtentStore( + const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore( + extentMetadataStore, + logger + ) : new FSExtentStore( extentMetadataStore, configuration.persistencePathArray, logger diff --git a/src/blob/SqlBlobServer.ts b/src/blob/SqlBlobServer.ts index c0e07e6d3..b36055732 100644 --- a/src/blob/SqlBlobServer.ts +++ b/src/blob/SqlBlobServer.ts @@ -8,6 +8,7 @@ import IGCManager from "../common/IGCManager"; import IRequestListenerFactory from "../common/IRequestListenerFactory"; import logger from "../common/Logger"; import FSExtentStore from "../common/persistence/FSExtentStore"; +import MemoryExtentStore from "../common/persistence/MemoryExtentStore"; import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; import IExtentStore from "../common/persistence/IExtentStore"; import SqlExtentMetadataStore from "../common/persistence/SqlExtentMetadataStore"; @@ -76,7 +77,10 @@ export default class SqlBlobServer extends ServerBase { configuration.sequelizeOptions ); - const extentStore: IExtentStore = new FSExtentStore( + const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore( + extentMetadataStore, + logger + ) : new FSExtentStore( extentMetadataStore, configuration.persistenceArray, logger diff --git a/src/blob/persistence/LokiBlobMetadataStore.ts b/src/blob/persistence/LokiBlobMetadataStore.ts index ebf53cbfc..fb1b10308 100644 --- a/src/blob/persistence/LokiBlobMetadataStore.ts +++ b/src/blob/persistence/LokiBlobMetadataStore.ts @@ -105,8 +105,11 @@ export default class LokiBlobMetadataStore private readonly pageBlobRangesManager = new PageBlobRangesManager(); - public constructor(public readonly lokiDBPath: string) { - this.db = new Loki(lokiDBPath, { + public constructor(public readonly lokiDBPath: string, inMemory: boolean) { + this.db = new Loki(lokiDBPath, inMemory ? { + persistenceMethod: "memory" + } : { + persistenceMethod: "fs", autosave: true, autosaveInterval: 5000 }); diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index 42336eec7..b1f634473 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -22,6 +22,7 @@ export default abstract class ConfigurationBase { public readonly pwd: string = "", public readonly oauth?: string, public readonly disableProductStyleUrl: boolean = false, + public readonly isMemoryPersistence: boolean = false, ) {} public hasCert() { diff --git a/src/common/persistence/LokiExtentMetadataStore.ts b/src/common/persistence/LokiExtentMetadataStore.ts index 28c2dca16..bad95e27c 100644 --- a/src/common/persistence/LokiExtentMetadataStore.ts +++ b/src/common/persistence/LokiExtentMetadataStore.ts @@ -25,8 +25,11 @@ export default class LokiExtentMetadata implements IExtentMetadataStore { private readonly EXTENTS_COLLECTION = "$EXTENTS_COLLECTION$"; - public constructor(public readonly lokiDBPath: string) { - this.db = new Loki(lokiDBPath, { + public constructor(public readonly lokiDBPath: string, inMemory: boolean) { + this.db = new Loki(lokiDBPath, inMemory ? { + persistenceMethod: "memory" + } : { + persistenceMethod: "fs", autosave: true, autosaveInterval: 5000 }); diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts new file mode 100644 index 000000000..fb824dcaf --- /dev/null +++ b/src/common/persistence/MemoryExtentStore.ts @@ -0,0 +1,190 @@ +import { ZERO_EXTENT_ID } from "../../blob/persistence/IBlobMetadataStore"; +import ILogger from "../ILogger"; +import ZeroBytesStream from "../ZeroBytesStream"; +import IExtentMetadataStore, { IExtentModel } from "./IExtentMetadataStore"; +import IExtentStore, { IExtentChunk } from "./IExtentStore"; +import uuid = require("uuid"); +import MemoryStream from 'memorystream' +import multistream = require("multistream"); + +export interface IMemoryExtentChunk extends IExtentChunk { + chunks: (Buffer | string)[] +} + +export default class MemoryExtentStore implements IExtentStore { + private readonly metadataStore: IExtentMetadataStore; + private readonly logger: ILogger; + private readonly chunks: Map = new Map(); + + private initialized: boolean = false; + private closed: boolean = true; + + public constructor( + metadata: IExtentMetadataStore, + logger: ILogger + ) { + this.metadataStore = metadata; + this.logger = logger; + } + + public isInitialized(): boolean { + return this.initialized; + } + + public isClosed(): boolean { + return this.closed; + } + + async init(): Promise { + if (!this.metadataStore.isInitialized()) { + await this.metadataStore.init(); + } + + this.initialized = true; + this.closed = false; + } + + public async close(): Promise { + if (!this.metadataStore.isClosed()) { + await this.metadataStore.close(); + } + + this.closed = true; + } + + public async clean(): Promise { + if (this.isClosed()) { + this.chunks.clear(); + return; + } + throw new Error(`Cannot clean MemoryExtentStore, it's not closed.`); + } + + async appendExtent(data: NodeJS.ReadableStream | Buffer, contextId?: string | undefined): Promise { + const chunks: (Buffer | string)[] = [] + let count = 0; + if (data instanceof Buffer) { + chunks.push(data) + count = data.length + } else { + for await (let chunk of data) { + chunks.push(chunk) + count += chunk.length + } + } + + const extentChunk: IMemoryExtentChunk = { + count, + offset: 0, + id: uuid(), + chunks + } + + this.chunks.set(extentChunk.id, extentChunk); + + const extent: IExtentModel = { + id: extentChunk.id, + locationId: extentChunk.id, + path: extentChunk.id, + size: count, + lastModifiedInMS: Date.now() + }; + + await this.metadataStore.updateExtent(extent); + + return extentChunk + } + + async readExtent(extentChunk?: IExtentChunk | undefined, contextId?: string | undefined): Promise { + if (extentChunk === undefined || extentChunk.count === 0) { + return new ZeroBytesStream(0); + } + + if (extentChunk.id === ZERO_EXTENT_ID) { + const subRangeCount = Math.min(extentChunk.count); + return new ZeroBytesStream(subRangeCount); + } + + const match = this.chunks.get(extentChunk.id); + if (!match) { + throw new Error(`Extend ${extentChunk.id} does not exist.`); + } + + return new MemoryStream(match.chunks); + } + + async readExtents(extentChunkArray: IExtentChunk[], offset: number, count: number, contextId?: string | undefined): Promise { + this.logger.verbose( + `MemoryExtentStore:readExtents() Start read from multi extents...`, + contextId + ); + + if (count === 0) { + return new ZeroBytesStream(0); + } + + const start = offset; // Start inclusive position in the merged stream + const end = offset + count; // End exclusive position in the merged stream + + const streams: NodeJS.ReadableStream[] = []; + let accumulatedOffset = 0; // Current payload offset in the merged stream + + for (const chunk of extentChunkArray) { + const nextOffset = accumulatedOffset + chunk.count; + + if (nextOffset <= start) { + accumulatedOffset = nextOffset; + continue; + } else if (end <= accumulatedOffset) { + break; + } else { + let chunkStart = chunk.offset; + let chunkEnd = chunk.offset + chunk.count; + if (start > accumulatedOffset) { + chunkStart = chunkStart + start - accumulatedOffset; // Inclusive + } + + if (end <= nextOffset) { + chunkEnd = chunkEnd - (nextOffset - end); // Exclusive + } + + streams.push( + await this.readExtent( + { + id: chunk.id, + offset: chunkStart, + count: chunkEnd - chunkStart + }, + contextId + ) + ); + accumulatedOffset = nextOffset; + } + } + + // TODO: What happens when count exceeds merged payload length? + // throw an error of just return as much data as we can? + if (end !== Infinity && accumulatedOffset < end) { + throw new RangeError( + // tslint:disable-next-line:max-line-length + `Not enough payload data error. Total length of payloads is ${accumulatedOffset}, while required data offset is ${offset}, count is ${count}.` + ); + } + + return multistream(streams); + } + + async deleteExtents(extents: Iterable): Promise { + let count = 0; + for (const id of extents) { + this.chunks.delete(id) + await this.metadataStore.deleteExtent(id); + count++; + } + return count; + } + + getMetadataStore(): IExtentMetadataStore { + return this.metadataStore; + } +} \ No newline at end of file diff --git a/src/queue/QueueServer.ts b/src/queue/QueueServer.ts index 50d4758dc..2e3bab020 100644 --- a/src/queue/QueueServer.ts +++ b/src/queue/QueueServer.ts @@ -8,6 +8,7 @@ import IGCManager from "../common/IGCManager"; import IRequestListenerFactory from "../common/IRequestListenerFactory"; import logger from "../common/Logger"; import FSExtentStore from "../common/persistence/FSExtentStore"; +import MemoryExtentStore from "../common/persistence/MemoryExtentStore"; import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; import IExtentStore from "../common/persistence/IExtentStore"; import LokiExtentMetadataStore from "../common/persistence/LokiExtentMetadataStore"; @@ -72,15 +73,19 @@ export default class QueueServer extends ServerBase { // creating a new XXXDataStore class implementing IBlobDataStore interface // and replace the default LokiBlobDataStore const metadataStore: IQueueMetadataStore = new LokiQueueMetadataStore( - configuration.metadataDBPath - // logger + configuration.metadataDBPath, + configuration.isMemoryPersistence ); const extentMetadataStore = new LokiExtentMetadataStore( - configuration.extentDBPath + configuration.extentDBPath, + configuration.isMemoryPersistence ); - const extentStore: IExtentStore = new FSExtentStore( + const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore( + extentMetadataStore, + logger + ) : new FSExtentStore( extentMetadataStore, configuration.persistencePathArray, logger diff --git a/src/queue/persistence/LokiQueueMetadataStore.ts b/src/queue/persistence/LokiQueueMetadataStore.ts index e4f6630dc..dea44ad49 100644 --- a/src/queue/persistence/LokiQueueMetadataStore.ts +++ b/src/queue/persistence/LokiQueueMetadataStore.ts @@ -50,8 +50,11 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore { private readonly QUEUES_COLLECTION = "$QUEUES_COLLECTION$"; private readonly MESSAGES_COLLECTION = "$MESSAGES_COLLECTION$"; - public constructor(public readonly lokiDBPath: string) { - this.db = new Loki(lokiDBPath, { + public constructor(public readonly lokiDBPath: string, inMemory: boolean) { + this.db = new Loki(lokiDBPath, inMemory ? { + persistenceMethod: "memory" + } : { + persistenceMethod: "fs", autosave: true, autosaveInterval: 5000 }); diff --git a/src/table/TableServer.ts b/src/table/TableServer.ts index 0968f3768..15975b99d 100644 --- a/src/table/TableServer.ts +++ b/src/table/TableServer.ts @@ -50,7 +50,8 @@ export default class TableServer extends ServerBase { // Create **dataStore with Loki.js const metadataStore: ITableMetadataStore = new LokiTableMetadataStore( - configuration.metadataDBPath + configuration.metadataDBPath, + configuration.isMemoryPersistence ); const accountDataStore: IAccountDataStore = new AccountDataStore(logger); diff --git a/src/table/persistence/LokiTableMetadataStore.ts b/src/table/persistence/LokiTableMetadataStore.ts index eedb44409..92b41b171 100644 --- a/src/table/persistence/LokiTableMetadataStore.ts +++ b/src/table/persistence/LokiTableMetadataStore.ts @@ -32,8 +32,11 @@ export default class LokiTableMetadataStore implements ITableMetadataStore { private transactionRollbackTheseEntities: Entity[] = []; // can maybe use Entity instead of any private transactionDeleteTheseEntities: Entity[] = []; // can maybe use Entity instead of any - public constructor(public readonly lokiDBPath: string) { - this.db = new Loki(lokiDBPath, { + public constructor(public readonly lokiDBPath: string, inMemory: boolean) { + this.db = new Loki(lokiDBPath, inMemory ? { + persistenceMethod: "memory" + } : { + persistenceMethod: "fs", autosave: true, autosaveInterval: 5000 }); From af85ace6c94497b87330f9ed36880d906d4cf9b9 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sun, 15 Oct 2023 15:20:02 -0400 Subject: [PATCH 02/37] Update config --- src/blob/BlobConfiguration.ts | 6 ++++-- src/blob/BlobEnvironment.ts | 7 +++++++ src/blob/BlobServerFactory.ts | 3 ++- src/blob/IBlobEnvironment.ts | 1 + src/blob/SqlBlobConfiguration.ts | 6 ++++-- src/common/Environment.ts | 11 +++++++++++ src/common/IEnvironment.ts | 4 +++- src/common/VSCEnvironment.ts | 4 ++++ src/common/VSCServerManagerBlob.ts | 3 ++- src/queue/IQueueEnvironment.ts | 1 + src/queue/QueueConfiguration.ts | 6 ++++-- src/queue/QueueEnvironment.ts | 7 +++++++ src/table/ITableEnvironment.ts | 2 ++ src/table/TableConfiguration.ts | 6 ++++-- src/table/TableEnvironment.ts | 7 +++++++ 15 files changed, 63 insertions(+), 11 deletions(-) diff --git a/src/blob/BlobConfiguration.ts b/src/blob/BlobConfiguration.ts index 9b9703cb4..619b0a399 100644 --- a/src/blob/BlobConfiguration.ts +++ b/src/blob/BlobConfiguration.ts @@ -39,7 +39,8 @@ export default class BlobConfiguration extends ConfigurationBase { key: string = "", pwd: string = "", oauth?: string, - disableProductStyleUrl: boolean = false + disableProductStyleUrl: boolean = false, + isMemoryPersistence: boolean = false, ) { super( host, @@ -54,7 +55,8 @@ export default class BlobConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl + disableProductStyleUrl, + isMemoryPersistence, ); } } diff --git a/src/blob/BlobEnvironment.ts b/src/blob/BlobEnvironment.ts index b5a54ebbb..f65a094aa 100644 --- a/src/blob/BlobEnvironment.ts +++ b/src/blob/BlobEnvironment.ts @@ -118,6 +118,13 @@ export default class BlobEnvironment implements IBlobEnvironment { return false; } + public inMemoryPersistence(): boolean { + if (this.flags.inMemoryPersistence !== undefined) { + return true; + } + return false; + } + public async debug(): Promise { if (typeof this.flags.debug === "string") { // Enable debug log to file diff --git a/src/blob/BlobServerFactory.ts b/src/blob/BlobServerFactory.ts index fb076a65d..2096ddb7a 100644 --- a/src/blob/BlobServerFactory.ts +++ b/src/blob/BlobServerFactory.ts @@ -79,7 +79,8 @@ export class BlobServerFactory { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); return new BlobServer(config); } diff --git a/src/blob/IBlobEnvironment.ts b/src/blob/IBlobEnvironment.ts index 822e770e8..06bd7342b 100644 --- a/src/blob/IBlobEnvironment.ts +++ b/src/blob/IBlobEnvironment.ts @@ -11,4 +11,5 @@ export default interface IBlobEnvironment { debug(): Promise; oauth(): string | undefined; disableProductStyleUrl(): boolean; + inMemoryPersistence(): boolean; } diff --git a/src/blob/SqlBlobConfiguration.ts b/src/blob/SqlBlobConfiguration.ts index a6e8b0586..831a62335 100644 --- a/src/blob/SqlBlobConfiguration.ts +++ b/src/blob/SqlBlobConfiguration.ts @@ -35,7 +35,8 @@ export default class SqlBlobConfiguration extends ConfigurationBase { key: string = "", pwd: string = "", oauth?: string, - disableProductStyleUrl: boolean = false + disableProductStyleUrl: boolean = false, + isMemoryPersistence: boolean = false, ) { super( host, @@ -50,7 +51,8 @@ export default class SqlBlobConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl + disableProductStyleUrl, + isMemoryPersistence, ); } } diff --git a/src/common/Environment.ts b/src/common/Environment.ts index 5b90ea6b9..abbaf2af8 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -66,6 +66,10 @@ args ["", "disableProductStyleUrl"], "Optional. Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri." ) + .option( + ["", "inMemoryPersistence"], + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + ) .option(["", "oauth"], 'Optional. OAuth level. Candidate values: "basic"') .option(["", "cert"], "Optional. Path to certificate file") .option(["", "key"], "Optional. Path to certificate key .pem file") @@ -155,6 +159,13 @@ export default class Environment implements IEnvironment { return this.flags.oauth; } + public inMemoryPersistence(): boolean { + if (this.flags.inMemoryPersistence !== undefined) { + return true; + } + return false; + } + public async debug(): Promise { if (typeof this.flags.debug === "string") { // Enable debug log to file diff --git a/src/common/IEnvironment.ts b/src/common/IEnvironment.ts index 5ae8acb99..e307b0a32 100644 --- a/src/common/IEnvironment.ts +++ b/src/common/IEnvironment.ts @@ -1,6 +1,8 @@ import IBlobEnvironment from "../blob/IBlobEnvironment"; import IQueueEnvironment from "../queue/IQueueEnvironment"; +import ITableEnvironment from "../table/ITableEnvironment"; export default interface IEnvironment extends IBlobEnvironment, - IQueueEnvironment {} + IQueueEnvironment, + ITableEnvironment { } diff --git a/src/common/VSCEnvironment.ts b/src/common/VSCEnvironment.ts index 179c6853d..246ea364a 100644 --- a/src/common/VSCEnvironment.ts +++ b/src/common/VSCEnvironment.ts @@ -109,4 +109,8 @@ export default class VSCEnvironment implements IEnvironment { this.workspaceConfiguration.get("disableProductStyleUrl") || false ); } + + public inMemoryPersistence(): boolean { + return this.workspaceConfiguration.get("inMemoryPersistence") || false; + } } diff --git a/src/common/VSCServerManagerBlob.ts b/src/common/VSCServerManagerBlob.ts index 03146322e..e4c497457 100644 --- a/src/common/VSCServerManagerBlob.ts +++ b/src/common/VSCServerManagerBlob.ts @@ -87,7 +87,8 @@ export default class VSCServerManagerBlob extends VSCServerManagerBase { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); return config; } diff --git a/src/queue/IQueueEnvironment.ts b/src/queue/IQueueEnvironment.ts index e15bf3413..7d2f1f861 100644 --- a/src/queue/IQueueEnvironment.ts +++ b/src/queue/IQueueEnvironment.ts @@ -10,4 +10,5 @@ export default interface IQueueEnvironment { key(): string | undefined; pwd(): string | undefined; debug(): Promise; + inMemoryPersistence(): boolean; } diff --git a/src/queue/QueueConfiguration.ts b/src/queue/QueueConfiguration.ts index 39fe3802b..0eed1e2f8 100644 --- a/src/queue/QueueConfiguration.ts +++ b/src/queue/QueueConfiguration.ts @@ -39,7 +39,8 @@ export default class QueueConfiguration extends ConfigurationBase { key: string = "", pwd: string = "", oauth?: string, - disableProductStyleUrl: boolean = false + disableProductStyleUrl: boolean = false, + isMemoryPersistence: boolean = false, ) { super( host, @@ -54,7 +55,8 @@ export default class QueueConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl + disableProductStyleUrl, + isMemoryPersistence, ); } } diff --git a/src/queue/QueueEnvironment.ts b/src/queue/QueueEnvironment.ts index 8a6d0b8fc..b1d1f5887 100644 --- a/src/queue/QueueEnvironment.ts +++ b/src/queue/QueueEnvironment.ts @@ -108,6 +108,13 @@ export default class QueueEnvironment implements IQueueEnvironment { return false; } + public inMemoryPersistence(): boolean { + if (this.flags.inMemoryPersistence !== undefined) { + return true; + } + return false; + } + public async debug(): Promise { if (typeof this.flags.debug === "string") { // Enable debug log to file diff --git a/src/table/ITableEnvironment.ts b/src/table/ITableEnvironment.ts index 79dbbd24e..bbd0b2374 100644 --- a/src/table/ITableEnvironment.ts +++ b/src/table/ITableEnvironment.ts @@ -20,4 +20,6 @@ export default interface ITableEnvironment { disableProductStyleUrl(): boolean; /** Optional. Enable debug log by providing a valid local file, path as log destination path as log destination */ debug(): Promise; + /** Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost */ + inMemoryPersistence(): boolean; } diff --git a/src/table/TableConfiguration.ts b/src/table/TableConfiguration.ts index a63fcb4d5..ac3980385 100644 --- a/src/table/TableConfiguration.ts +++ b/src/table/TableConfiguration.ts @@ -35,7 +35,8 @@ export default class TableConfiguration extends ConfigurationBase { key: string = "", pwd: string = "", oauth?: string, - disableProductStyleUrl: boolean = false + disableProductStyleUrl: boolean = false, + isMemoryPersistence: boolean = false, ) { super( host, @@ -50,7 +51,8 @@ export default class TableConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl + disableProductStyleUrl, + isMemoryPersistence, ); } } diff --git a/src/table/TableEnvironment.ts b/src/table/TableEnvironment.ts index 5e016ae89..9243113b5 100644 --- a/src/table/TableEnvironment.ts +++ b/src/table/TableEnvironment.ts @@ -100,6 +100,13 @@ export default class TableEnvironment implements ITableEnvironment { return false; } + public inMemoryPersistence(): boolean { + if (this.flags.inMemoryPersistence !== undefined) { + return true; + } + return false; + } + public async debug(): Promise { if (typeof this.flags.debug === "string") { // Enable debug log to file From b3846f88e4bbb7433e63a5ef126893787eda680f Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sun, 15 Oct 2023 15:23:11 -0400 Subject: [PATCH 03/37] Plumb config --- src/azurite.ts | 6 ++++-- src/blob/BlobServerFactory.ts | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/azurite.ts b/src/azurite.ts index 5801d5481..1b156b7df 100644 --- a/src/azurite.ts +++ b/src/azurite.ts @@ -98,7 +98,8 @@ async function main() { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); const tableConfig = new TableConfiguration( @@ -115,7 +116,8 @@ async function main() { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); // We use logger singleton as global debugger logger to track detailed outputs cross layers diff --git a/src/blob/BlobServerFactory.ts b/src/blob/BlobServerFactory.ts index 2096ddb7a..8d5b4a181 100644 --- a/src/blob/BlobServerFactory.ts +++ b/src/blob/BlobServerFactory.ts @@ -58,7 +58,8 @@ export class BlobServerFactory { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); return new SqlBlobServer(config); From 6532e5a6663884da6e3c867ca2df046fb92600da Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sun, 15 Oct 2023 16:05:00 -0400 Subject: [PATCH 04/37] Switch to newer buffer stream library --- .vscode/launch.json | 13 ++++ package-lock.json | 66 ++++++++++----------- package.json | 4 +- src/common/persistence/MemoryExtentStore.ts | 8 ++- src/queue/handlers/MessagesHandler.ts | 1 - 5 files changed, 54 insertions(+), 38 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 4eb29e85b..0aa75a80a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,6 +17,19 @@ "skipFiles": ["node_modules/*/**", "/*/**"], "outputCapture": "std" }, + { + "type": "node", + "request": "launch", + "name": "Azurite Service - Loki, in-memory", + "cwd": "${workspaceFolder}", + "runtimeArgs": ["-r", "ts-node/register"], + "args": ["${workspaceFolder}/src/azurite.ts", "-d", "debug.log", "--inMemoryPersistence"], + "env": { + "AZURITE_ACCOUNTS": "" + }, + "skipFiles": ["node_modules/*/**", "/*/**"], + "outputCapture": "std" + }, { "type": "node", "request": "launch", diff --git a/package-lock.json b/package-lock.json index 3384478f0..95aa7c05b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,13 +18,13 @@ "glob-to-regexp": "^0.4.1", "jsonwebtoken": "^9.0.0", "lokijs": "^1.5.6", - "memorystream": "^0.3.1", "morgan": "^1.9.1", "multistream": "^2.1.1", "mysql2": "^3.2.0", "rimraf": "^3.0.2", "sequelize": "^6.31.0", "stoppable": "^1.1.0", + "stream-buffers": "^3.0.2", "tedious": "^16.0.0", "to-readable-stream": "^2.1.0", "tslib": "^2.3.0", @@ -54,13 +54,13 @@ "@types/glob-to-regexp": "^0.4.1", "@types/jsonwebtoken": "^9.0.1", "@types/lokijs": "^1.5.2", - "@types/memorystream": "^0.3.2", "@types/mocha": "^9.0.0", "@types/morgan": "^1.7.35", "@types/multistream": "^2.1.1", "@types/node": "^14.14.24", "@types/rimraf": "^3.0.0", "@types/stoppable": "^1.1.1", + "@types/stream-buffers": "^3.0.5", "@types/uri-templates": "^0.1.29", "@types/uuid": "^3.4.4", "@types/validator": "^13.1.4", @@ -1456,15 +1456,6 @@ "integrity": "sha512-Q/F6OUCZPHWY4hzEowhCswi9Tafc/E7DCUyyWIOH3+hM3K96Mkj2U3byfzs7Yd542I8gT/8oUALnoddqdA20xg==", "dev": true }, - "node_modules/@types/memorystream": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@types/memorystream/-/memorystream-0.3.2.tgz", - "integrity": "sha512-92v1riJ4LA00mKsNf9Ac8ubeOBB7zEO+L+ZCOrd22QSsEUVEy2be2GlZLBt7NKJQBguGBSVhZSuaKf/zATVClw==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/mime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", @@ -1560,6 +1551,15 @@ "@types/node": "*" } }, + "node_modules/@types/stream-buffers": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/stream-buffers/-/stream-buffers-3.0.5.tgz", + "integrity": "sha512-mEKCVRgR+0fM3FUcywARE6J50sz++u/KQL5sBMJKFzLTyX/WsKN2THr7vMic6fWoFfT454LXOnJYUMp8ANNBMw==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -7200,14 +7200,6 @@ "node": ">= 0.6" } }, - "node_modules/memorystream": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", - "integrity": "sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==", - "engines": { - "node": ">= 0.10.0" - } - }, "node_modules/merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -9271,6 +9263,14 @@ "npm": ">=6" } }, + "node_modules/stream-buffers": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.2.tgz", + "integrity": "sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==", + "engines": { + "node": ">= 0.10.0" + } + }, "node_modules/stream-meter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz", @@ -11441,15 +11441,6 @@ "integrity": "sha512-Q/F6OUCZPHWY4hzEowhCswi9Tafc/E7DCUyyWIOH3+hM3K96Mkj2U3byfzs7Yd542I8gT/8oUALnoddqdA20xg==", "dev": true }, - "@types/memorystream": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@types/memorystream/-/memorystream-0.3.2.tgz", - "integrity": "sha512-92v1riJ4LA00mKsNf9Ac8ubeOBB7zEO+L+ZCOrd22QSsEUVEy2be2GlZLBt7NKJQBguGBSVhZSuaKf/zATVClw==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/mime": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/@types/mime/-/mime-2.0.1.tgz", @@ -11545,6 +11536,15 @@ "@types/node": "*" } }, + "@types/stream-buffers": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/stream-buffers/-/stream-buffers-3.0.5.tgz", + "integrity": "sha512-mEKCVRgR+0fM3FUcywARE6J50sz++u/KQL5sBMJKFzLTyX/WsKN2THr7vMic6fWoFfT454LXOnJYUMp8ANNBMw==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -15881,11 +15881,6 @@ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" }, - "memorystream": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", - "integrity": "sha512-S3UwM3yj5mtUSEfP41UZmt/0SCoVYUcU1rkXv+BQ5Ig8ndL4sPoJNBUJERafdPb5jjHJGuMgytgKvKIf58XNBw==" - }, "merge-descriptors": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", @@ -17378,6 +17373,11 @@ "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==" }, + "stream-buffers": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.2.tgz", + "integrity": "sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==" + }, "stream-meter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz", diff --git a/package.json b/package.json index d9d7642b0..616b710e1 100644 --- a/package.json +++ b/package.json @@ -29,13 +29,13 @@ "glob-to-regexp": "^0.4.1", "jsonwebtoken": "^9.0.0", "lokijs": "^1.5.6", - "memorystream": "^0.3.1", "morgan": "^1.9.1", "multistream": "^2.1.1", "mysql2": "^3.2.0", "rimraf": "^3.0.2", "sequelize": "^6.31.0", "stoppable": "^1.1.0", + "stream-buffers": "^3.0.2", "tedious": "^16.0.0", "to-readable-stream": "^2.1.0", "tslib": "^2.3.0", @@ -59,13 +59,13 @@ "@types/glob-to-regexp": "^0.4.1", "@types/jsonwebtoken": "^9.0.1", "@types/lokijs": "^1.5.2", - "@types/memorystream": "^0.3.2", "@types/mocha": "^9.0.0", "@types/morgan": "^1.7.35", "@types/multistream": "^2.1.1", "@types/node": "^14.14.24", "@types/rimraf": "^3.0.0", "@types/stoppable": "^1.1.1", + "@types/stream-buffers": "^3.0.5", "@types/uri-templates": "^0.1.29", "@types/uuid": "^3.4.4", "@types/validator": "^13.1.4", diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts index fb824dcaf..ef6974fa1 100644 --- a/src/common/persistence/MemoryExtentStore.ts +++ b/src/common/persistence/MemoryExtentStore.ts @@ -4,7 +4,7 @@ import ZeroBytesStream from "../ZeroBytesStream"; import IExtentMetadataStore, { IExtentModel } from "./IExtentMetadataStore"; import IExtentStore, { IExtentChunk } from "./IExtentStore"; import uuid = require("uuid"); -import MemoryStream from 'memorystream' +import { ReadableStreamBuffer } from 'stream-buffers' import multistream = require("multistream"); export interface IMemoryExtentChunk extends IExtentChunk { @@ -110,7 +110,11 @@ export default class MemoryExtentStore implements IExtentStore { throw new Error(`Extend ${extentChunk.id} does not exist.`); } - return new MemoryStream(match.chunks); + const buffer = new ReadableStreamBuffer(); + match.chunks.forEach(chunk => buffer.put(chunk)) + buffer.stop(); + + return buffer; } async readExtents(extentChunkArray: IExtentChunk[], offset: number, count: number, contextId?: string | undefined): Promise { diff --git a/src/queue/handlers/MessagesHandler.ts b/src/queue/handlers/MessagesHandler.ts index 5c1002d89..c54755688 100644 --- a/src/queue/handlers/MessagesHandler.ts +++ b/src/queue/handlers/MessagesHandler.ts @@ -116,7 +116,6 @@ export default class MessagesHandler extends BaseHandler statusCode: 200; }; - // Read the message text from file system. for (const message of messages) { const textStream = await this.extentStore.readExtent( message.persistency, From d00464ee5e21a29e8708984417daefabc2c0e5dd Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sun, 15 Oct 2023 16:51:48 -0400 Subject: [PATCH 05/37] Add the config in more places and update markdowns --- ChangeLog.md | 4 ++++ README.mcr.md | 2 ++ README.md | 13 +++++++++++++ package.json | 7 ++++++- src/blob/BlobEnvironment.ts | 4 ++++ src/common/Environment.ts | 8 ++++---- src/queue/QueueEnvironment.ts | 4 ++++ src/table/TableEnvironment.ts | 4 ++++ tests/BlobTestServerFactory.ts | 11 ++++++++--- 9 files changed, 49 insertions(+), 8 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index a96f0f198..6a5a6c4c5 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -4,6 +4,10 @@ ## Upcoming Release +General: + +- Add `--inMemoryStorage` option and related configs to persist all data in-memory without disk persistence. (issue #2227) + Blob: - Fix validation of Blob SAS token when using the second key for an account in `AZURITE_ACCOUNTS` diff --git a/README.mcr.md b/README.mcr.md index 531c4cfd1..f9bd7fc8c 100644 --- a/README.mcr.md +++ b/README.mcr.md @@ -74,6 +74,8 @@ Above command will try to start Azurite image with configurations: `--disableProductStyleUrl` force parsing storage account name from request Uri path, instead of from request Uri host. +`--inMemoryPersistence` disable persisting any data to disk. If the Azurite process is terminated, all data is lost. + > If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. > In above sample, you need to use **double first forward slash** for location and debug path parameters to avoid a [known issue](https://stackoverflow.com/questions/48427366/docker-build-command-add-c-program-files-git-to-the-path-passed-as-build-argu) for Git on Windows. diff --git a/README.md b/README.md index 8db16d5ba..7991aa55b 100644 --- a/README.md +++ b/README.md @@ -198,6 +198,7 @@ Following extension configurations are supported: - `azurite.oauth` OAuth oauthentication level. Candidate level values: `basic`. - `azurite.skipApiVersionCheck` Skip the request API version check, by default false. - `azurite.disableProductStyleUrl` Force parsing storage account name from request Uri path, instead of from request Uri host. +- `azurite.inMemoryPersistence` Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. ### [DockerHub](https://hub.docker.com/_/microsoft-azure-storage-azurite) @@ -257,6 +258,8 @@ Above command will try to start Azurite image with configurations: `--disableProductStyleUrl` force parsing storage account name from request Uri path, instead of from request Uri host. +`--inMemoryPersistence` disable persisting any data to disk. If the Azurite process is terminated, all data is lost. + > If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. > In above sample, you need to use **double first forward slash** for location and debug path parameters to avoid a [known issue](https://stackoverflow.com/questions/48427366/docker-build-command-add-c-program-files-git-to-the-path-passed-as-build-argu) for Git on Windows. @@ -430,6 +433,16 @@ Optional. When using FQDN instead of IP in request Uri host, by default Azurite --disableProductStyleUrl ``` +### Use in-memory storage + +Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. +By default, LokiJS persists blob and queue metadata to disk and content to extent files. Table storage +persists all data to disk. This behavior can be disabled using this option. + +```cmd +--inMemoryStorage +``` + ### Command Line Options Differences between Azurite V2 Azurite V3 supports SharedKey, Account Shared Access Signature (SAS), Service SAS, OAuth, and Public Container Access authentications, you can use any Azure Storage SDKs or tools like Storage Explorer to connect Azurite V3 with any authentication strategy. diff --git a/package.json b/package.json index 616b710e1..dddf91e01 100644 --- a/package.json +++ b/package.json @@ -250,6 +250,11 @@ "type": "boolean", "default": false, "description": "Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri." + }, + "azurite.inMemoryStorage": { + "type": "boolean", + "default": false, + "description": "Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." } } } @@ -322,4 +327,4 @@ "url": "https://github.com/azure/azurite/issues" }, "homepage": "https://github.com/azure/azurite#readme" -} +} \ No newline at end of file diff --git a/src/blob/BlobEnvironment.ts b/src/blob/BlobEnvironment.ts index f65a094aa..ee82d9e33 100644 --- a/src/blob/BlobEnvironment.ts +++ b/src/blob/BlobEnvironment.ts @@ -40,6 +40,10 @@ if (!(args as any).config.name) { .option(["", "oauth"], 'Optional. OAuth level. Candidate values: "basic"') .option(["", "cert"], "Optional. Path to certificate file") .option(["", "key"], "Optional. Path to certificate key .pem file") + .option( + ["", "inMemoryPersistence"], + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + ) .option( ["d", "debug"], "Optional. Enable debug log by providing a valid local file path as log destination" diff --git a/src/common/Environment.ts b/src/common/Environment.ts index abbaf2af8..3d2968710 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -66,14 +66,14 @@ args ["", "disableProductStyleUrl"], "Optional. Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri." ) - .option( - ["", "inMemoryPersistence"], - "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." - ) .option(["", "oauth"], 'Optional. OAuth level. Candidate values: "basic"') .option(["", "cert"], "Optional. Path to certificate file") .option(["", "key"], "Optional. Path to certificate key .pem file") .option(["", "pwd"], "Optional. Password for .pfx file") + .option( + ["", "inMemoryPersistence"], + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + ) .option( ["d", "debug"], "Optional. Enable debug log by providing a valid local file path as log destination" diff --git a/src/queue/QueueEnvironment.ts b/src/queue/QueueEnvironment.ts index b1d1f5887..ec7c22a32 100644 --- a/src/queue/QueueEnvironment.ts +++ b/src/queue/QueueEnvironment.ts @@ -39,6 +39,10 @@ args .option(["", "cert"], "Optional. Path to certificate file") .option(["", "key"], "Optional. Path to certificate key .pem file") .option(["", "pwd"], "Optional. Password for .pfx file") + .option( + ["", "inMemoryPersistence"], + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + ) .option( ["d", "debug"], "Optional. Enable debug log by providing a valid local file path as log destination" diff --git a/src/table/TableEnvironment.ts b/src/table/TableEnvironment.ts index 9243113b5..0447b4eaf 100644 --- a/src/table/TableEnvironment.ts +++ b/src/table/TableEnvironment.ts @@ -38,6 +38,10 @@ args ["", "disableProductStyleUrl"], "Optional. Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri." ) + .option( + ["", "inMemoryPersistence"], + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + ) .option( ["d", "debug"], "Optional. Enable debug log by providing a valid local file path as log destination" diff --git a/tests/BlobTestServerFactory.ts b/tests/BlobTestServerFactory.ts index c37c19848..58ca47fd4 100644 --- a/tests/BlobTestServerFactory.ts +++ b/tests/BlobTestServerFactory.ts @@ -10,7 +10,8 @@ export default class BlobTestServerFactory { loose: boolean = false, skipApiVersionCheck: boolean = false, https: boolean = false, - oauth?: string + oauth?: string, + inMemoryPersistence: boolean = false ): BlobServer | SqlBlobServer { const databaseConnectionString = process.env.AZURITE_TEST_DB; const isSQL = databaseConnectionString !== undefined; @@ -43,7 +44,9 @@ export default class BlobTestServerFactory { cert, key, undefined, - oauth + oauth, + undefined, + inMemoryPersistence ); return new SqlBlobServer(config); @@ -65,7 +68,9 @@ export default class BlobTestServerFactory { cert, key, undefined, - oauth + oauth, + undefined, + inMemoryPersistence ); return new BlobServer(config); } From 58aa6f086ec28e104ef500c44923a6ff17076a83 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sun, 15 Oct 2023 22:04:59 -0400 Subject: [PATCH 06/37] Use built-in stream --- package-lock.json | 33 --------------------- package.json | 7 ----- src/common/persistence/MemoryExtentStore.ts | 8 ++--- 3 files changed, 4 insertions(+), 44 deletions(-) diff --git a/package-lock.json b/package-lock.json index 95aa7c05b..edcbe3e15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,7 +24,6 @@ "rimraf": "^3.0.2", "sequelize": "^6.31.0", "stoppable": "^1.1.0", - "stream-buffers": "^3.0.2", "tedious": "^16.0.0", "to-readable-stream": "^2.1.0", "tslib": "^2.3.0", @@ -60,7 +59,6 @@ "@types/node": "^14.14.24", "@types/rimraf": "^3.0.0", "@types/stoppable": "^1.1.1", - "@types/stream-buffers": "^3.0.5", "@types/uri-templates": "^0.1.29", "@types/uuid": "^3.4.4", "@types/validator": "^13.1.4", @@ -1551,15 +1549,6 @@ "@types/node": "*" } }, - "node_modules/@types/stream-buffers": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@types/stream-buffers/-/stream-buffers-3.0.5.tgz", - "integrity": "sha512-mEKCVRgR+0fM3FUcywARE6J50sz++u/KQL5sBMJKFzLTyX/WsKN2THr7vMic6fWoFfT454LXOnJYUMp8ANNBMw==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -9263,14 +9252,6 @@ "npm": ">=6" } }, - "node_modules/stream-buffers": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.2.tgz", - "integrity": "sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==", - "engines": { - "node": ">= 0.10.0" - } - }, "node_modules/stream-meter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz", @@ -11536,15 +11517,6 @@ "@types/node": "*" } }, - "@types/stream-buffers": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@types/stream-buffers/-/stream-buffers-3.0.5.tgz", - "integrity": "sha512-mEKCVRgR+0fM3FUcywARE6J50sz++u/KQL5sBMJKFzLTyX/WsKN2THr7vMic6fWoFfT454LXOnJYUMp8ANNBMw==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/tunnel": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/tunnel/-/tunnel-0.0.3.tgz", @@ -17373,11 +17345,6 @@ "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==" }, - "stream-buffers": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/stream-buffers/-/stream-buffers-3.0.2.tgz", - "integrity": "sha512-DQi1h8VEBA/lURbSwFtEHnSTb9s2/pwLEaFuNhXwy1Dx3Sa0lOuYT2yNUr4/j2fs8oCAMANtrZ5OrPZtyVs3MQ==" - }, "stream-meter": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/stream-meter/-/stream-meter-1.0.4.tgz", diff --git a/package.json b/package.json index dddf91e01..952136c2f 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,6 @@ "rimraf": "^3.0.2", "sequelize": "^6.31.0", "stoppable": "^1.1.0", - "stream-buffers": "^3.0.2", "tedious": "^16.0.0", "to-readable-stream": "^2.1.0", "tslib": "^2.3.0", @@ -65,7 +64,6 @@ "@types/node": "^14.14.24", "@types/rimraf": "^3.0.0", "@types/stoppable": "^1.1.1", - "@types/stream-buffers": "^3.0.5", "@types/uri-templates": "^0.1.29", "@types/uuid": "^3.4.4", "@types/validator": "^13.1.4", @@ -250,11 +248,6 @@ "type": "boolean", "default": false, "description": "Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri." - }, - "azurite.inMemoryStorage": { - "type": "boolean", - "default": false, - "description": "Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." } } } diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts index ef6974fa1..2ecab34d0 100644 --- a/src/common/persistence/MemoryExtentStore.ts +++ b/src/common/persistence/MemoryExtentStore.ts @@ -4,8 +4,8 @@ import ZeroBytesStream from "../ZeroBytesStream"; import IExtentMetadataStore, { IExtentModel } from "./IExtentMetadataStore"; import IExtentStore, { IExtentChunk } from "./IExtentStore"; import uuid = require("uuid"); -import { ReadableStreamBuffer } from 'stream-buffers' import multistream = require("multistream"); +import { Readable } from "stream"; export interface IMemoryExtentChunk extends IExtentChunk { chunks: (Buffer | string)[] @@ -110,9 +110,9 @@ export default class MemoryExtentStore implements IExtentStore { throw new Error(`Extend ${extentChunk.id} does not exist.`); } - const buffer = new ReadableStreamBuffer(); - match.chunks.forEach(chunk => buffer.put(chunk)) - buffer.stop(); + const buffer = new Readable() + match.chunks.forEach(chunk => buffer.push(chunk)) + buffer.push(null) return buffer; } From a255c7f70c5e592084f86684e6ebffe3c3e515f6 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Mon, 16 Oct 2023 23:50:19 -0400 Subject: [PATCH 07/37] Refactor table creation to a single factory --- tests/table/apis/table.entity.rest.test.ts | 28 ++---- .../table/apis/table.validation.rest.test.ts | 24 ++--- tests/table/auth/oauth.test.ts | 20 +++- tests/table/utils/TableTestServerFactory.ts | 38 +++++--- tests/table/utils/table.entity.test.utils.ts | 95 +++++++------------ 5 files changed, 95 insertions(+), 110 deletions(-) diff --git a/tests/table/apis/table.entity.rest.test.ts b/tests/table/apis/table.entity.rest.test.ts index b96f3a5a7..131090148 100644 --- a/tests/table/apis/table.entity.rest.test.ts +++ b/tests/table/apis/table.entity.rest.test.ts @@ -7,7 +7,6 @@ import * as assert from "assert"; import { AxiosResponse } from "axios"; import { configLogger } from "../../../src/common/Logger"; -import TableConfiguration from "../../../src/table/TableConfiguration"; import TableServer from "../../../src/table/TableServer"; import { getUniqueName } from "../../testutils"; import { createUniquePartitionKey } from "../utils/table.entity.test.utils"; @@ -19,35 +18,26 @@ import { postToAzurite, putToAzurite } from "../utils/table.entity.tests.rest.submitter"; +import TableTestServerFactory from "../utils/TableTestServerFactory"; // Set true to enable debug log configLogger(false); describe("table Entity APIs REST tests", () => { - // TODO: Create a server factory as tests utils - const host = "127.0.0.1"; - const port = 11002; - const metadataDbPath = "__tableTestsStorage__"; - const enableDebugLog: boolean = true; - const debugLogPath: string = ""; - const config = new TableConfiguration( - host, - port, - metadataDbPath, - enableDebugLog, - false, - undefined, - debugLogPath, - false, - true - ); let server: TableServer; let reproFlowsTableName: string = getUniqueName("flows"); before(async () => { - server = new TableServer(config); + server = new TableTestServerFactory().createServer({ + metadataDBPath: "__tableTestsStorage__", + enableDebugLog: true, + debugLogFilePath: "", + loose: false, + skipApiVersionCheck: true, + https: false + }); await server.start(); }); diff --git a/tests/table/apis/table.validation.rest.test.ts b/tests/table/apis/table.validation.rest.test.ts index 83117beb8..7199382c9 100644 --- a/tests/table/apis/table.validation.rest.test.ts +++ b/tests/table/apis/table.validation.rest.test.ts @@ -6,7 +6,6 @@ import * as assert from "assert"; import { configLogger } from "../../../src/common/Logger"; -import TableConfiguration from "../../../src/table/TableConfiguration"; import TableServer from "../../../src/table/TableServer"; import { getUniqueName } from "../../testutils"; import { @@ -14,34 +13,29 @@ import { getToAzurite, postToAzurite } from "../utils/table.entity.tests.rest.submitter"; +import TableTestServerFactory from "../utils/TableTestServerFactory"; // Set true to enable debug log configLogger(false); describe("table name validation tests", () => { - const host = "127.0.0.1"; - const port = 11002; const metadataDbPath = getUniqueName("__tableTestsStorage__"); const enableDebugLog: boolean = true; const debugLogPath: string = "g:/debug.log"; - const config = new TableConfiguration( - host, - port, - metadataDbPath, - enableDebugLog, - false, - undefined, - debugLogPath, - false, - true - ); let server: TableServer; let tableName: string = getUniqueName("flows"); before(async () => { - server = new TableServer(config); + server = new TableTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + enableDebugLog: enableDebugLog, + debugLogFilePath: debugLogPath, + loose: false, + skipApiVersionCheck: true, + https: false + }); await server.start(); }); diff --git a/tests/table/auth/oauth.test.ts b/tests/table/auth/oauth.test.ts index a2648d9fc..a87811ea9 100644 --- a/tests/table/auth/oauth.test.ts +++ b/tests/table/auth/oauth.test.ts @@ -13,7 +13,15 @@ configLogger(false); describe("Table OAuth Basic", () => { const factory = new TableTestServerFactory(); - let server = factory.createServer(false, false, true, "basic"); + let server = factory.createServer({ + metadataDBPath: "__test_db_table__.json", + enableDebugLog: false, + debugLogFilePath: "debug-test-table.log", + loose: false, + skipApiVersionCheck: false, + https: true, + oauth: "basic" + }); const baseURL = `https://${server.config.host}:${server.config.port}/devstoreaccount1`; before(async () => { @@ -386,7 +394,15 @@ describe("Table OAuth Basic", () => { await server.close(); await server.clean(); - server = factory.createServer(false, false, false, "basic"); + server = factory.createServer({ + metadataDBPath: "__test_db_table__.json", + enableDebugLog: false, + debugLogFilePath: "debug-test-table.log", + loose: false, + skipApiVersionCheck: false, + https: false, + oauth: "basic" + }); await server.start(); const httpBaseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; diff --git a/tests/table/utils/TableTestServerFactory.ts b/tests/table/utils/TableTestServerFactory.ts index 4e6f15f31..7b97197ee 100644 --- a/tests/table/utils/TableTestServerFactory.ts +++ b/tests/table/utils/TableTestServerFactory.ts @@ -1,34 +1,42 @@ import TableConfiguration from "../../../src/table/TableConfiguration"; import TableServer from "../../../src/table/TableServer"; +export interface ITableTestServerFactoryParams { + metadataDBPath: string + enableDebugLog: boolean + debugLogFilePath: string + loose: boolean + skipApiVersionCheck: boolean + https: boolean + oauth?: string +} + export default class TableTestServerFactory { - public createServer( - loose: boolean = false, - skipApiVersionCheck: boolean = false, - https: boolean = false, - oauth?: string - ): TableServer { + public createServer(params: ITableTestServerFactoryParams): TableServer { + const inMemoryPersistence = process.env.AZURITE_TEST_INMEMORYPERSISTENCE !== undefined; + const port = 11002; const host = "127.0.0.1"; - const cert = https ? "tests/server.cert" : undefined; - const key = https ? "tests/server.key" : undefined; + const cert = params.https ? "tests/server.cert" : undefined; + const key = params.https ? "tests/server.key" : undefined; - const lokiMetadataDBPath = "__test_db_table__.json"; const config = new TableConfiguration( host, port, - lokiMetadataDBPath, - false, + params.metadataDBPath, + params.enableDebugLog, false, undefined, - "debug-test-table.log", - loose, - skipApiVersionCheck, + params.debugLogFilePath, + params.loose, + params.skipApiVersionCheck, cert, key, undefined, - oauth + params.oauth, + undefined, + inMemoryPersistence ); return new TableServer(config); } diff --git a/tests/table/utils/table.entity.test.utils.ts b/tests/table/utils/table.entity.test.utils.ts index c18ed8ed8..8a7bc1a17 100644 --- a/tests/table/utils/table.entity.test.utils.ts +++ b/tests/table/utils/table.entity.test.utils.ts @@ -5,13 +5,13 @@ import { } from "../../testutils"; import TableServer from "../../../src/table/TableServer"; -import TableConfiguration from "../../../src/table/TableConfiguration"; import { AzureNamedKeyCredential, AzureSASCredential, TableClient } from "@azure/data-tables"; import { copyFile } from "fs"; +import TableTestServerFactory, { ITableTestServerFactoryParams } from "./TableTestServerFactory"; export const PROTOCOL = "http"; export const HOST = "127.0.0.1"; @@ -32,30 +32,6 @@ const AZURITE_TABLE_BASE_URL = "AZURITE_TABLE_BASE_URL"; // Azure Pipelines need a unique name per test instance // const REPRO_DB_PATH = "./querydb.json"; -const config = new TableConfiguration( - HOST, - PORT, - metadataDbPath, - enableDebugLog, - false, - undefined, - debugLogPath -); - -const httpsConfig = new TableConfiguration( - HOST, - PORT, - metadataDbPath, - enableDebugLog, - false, - undefined, - debugLogPath, - false, - true, - "tests/server.cert", - "tests/server.key" -); - /** * Creates the Azurite TableServer used in Table API tests * @@ -63,11 +39,25 @@ const httpsConfig = new TableConfiguration( * @return {*} {TableServer} */ export function createTableServerForTest(): TableServer { - return new TableServer(config); + return new TableTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + enableDebugLog: enableDebugLog, + debugLogFilePath: debugLogPath, + loose: false, + skipApiVersionCheck: false, + https: false + }); } export function createTableServerForTestHttps(): TableServer { - return new TableServer(httpsConfig); + return new TableTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + enableDebugLog: enableDebugLog, + debugLogFilePath: debugLogPath, + loose: false, + skipApiVersionCheck: true, + https: true + }); } /** @@ -84,26 +74,19 @@ export function createTableServerForQueryTestHttps(): TableServer { const uniqueDBpath = "./" + uniqueDbName + ".json"; duplicateReproDBForTest(uniqueDBpath); const queryConfig = createQueryConfig(uniqueDBpath); - return new TableServer(queryConfig); + return new TableTestServerFactory().createServer(queryConfig); } export function createTableServerForTestOAuth(oauth?: string): TableServer { - const oAuthConfig = new TableConfiguration( - HOST, - PORT, - metadataDbPath, - enableDebugLog, - false, - undefined, - debugLogPath, - false, - true, - undefined, - undefined, - undefined, - oauth - ); - return new TableServer(oAuthConfig); + return new TableTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + enableDebugLog: enableDebugLog, + debugLogFilePath: debugLogPath, + loose: false, + skipApiVersionCheck: true, + https: false, + oauth: oauth + }); } /** @@ -219,19 +202,13 @@ function duplicateReproDBForTest(uniqueDBpath: string) { ); } -function createQueryConfig(uniqueDBpath: string): TableConfiguration { - const queryConfig = new TableConfiguration( - HOST, - PORT, - uniqueDBpath, // contains guid and binProp object from legacy schema DB - enableDebugLog, - false, - undefined, - debugLogPath, - false, - true, - "tests/server.cert", - "tests/server.key" - ); - return queryConfig; +function createQueryConfig(uniqueDBpath: string): ITableTestServerFactoryParams { + return { + metadataDBPath: uniqueDBpath, // contains guid and binProp object from legacy schema DB + enableDebugLog: enableDebugLog, + debugLogFilePath: debugLogPath, + loose: false, + skipApiVersionCheck: true, + https: true + }; } From 657879f7764e971c8bc6aaf47ada49660ab415d3 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Mon, 16 Oct 2023 23:50:37 -0400 Subject: [PATCH 08/37] Move in-memory persistence to an env var --- tests/BlobTestServerFactory.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/BlobTestServerFactory.ts b/tests/BlobTestServerFactory.ts index 58ca47fd4..769e32c18 100644 --- a/tests/BlobTestServerFactory.ts +++ b/tests/BlobTestServerFactory.ts @@ -10,11 +10,11 @@ export default class BlobTestServerFactory { loose: boolean = false, skipApiVersionCheck: boolean = false, https: boolean = false, - oauth?: string, - inMemoryPersistence: boolean = false + oauth?: string ): BlobServer | SqlBlobServer { const databaseConnectionString = process.env.AZURITE_TEST_DB; const isSQL = databaseConnectionString !== undefined; + const inMemoryPersistence = process.env.AZURITE_TEST_INMEMORYPERSISTENCE !== undefined; const port = 11000; const host = "127.0.0.1"; From fca212e27e202c28bf45ea2d9232cdf6e2429d05 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 00:21:14 -0400 Subject: [PATCH 09/37] Add QueueTestServerFactory to centralize test server creation --- src/common/VSCServerManagerQueue.ts | 3 +- src/common/VSCServerManagerTable.ts | 3 +- src/queue/handlers/MessageIdHandler.ts | 4 +- src/queue/handlers/MessagesHandler.ts | 8 +-- src/queue/main.ts | 3 +- src/queue/utils/constants.ts | 6 +-- tests/queue/apis/messageid.test.ts | 19 +++---- tests/queue/apis/messages.test.ts | 19 +++---- tests/queue/apis/queue.test.ts | 19 +++---- tests/queue/apis/queueService.test.ts | 36 +++++--------- tests/queue/https.test.ts | 27 +++------- tests/queue/oauth.test.ts | 55 ++++++--------------- tests/queue/queueAuthentication.test.ts | 19 +++---- tests/queue/queueCorsRequest.test.ts | 19 +++---- tests/queue/queueSas.test.ts | 19 +++---- tests/queue/queueSpecialnaming.test.ts | 19 +++---- tests/queue/utils/QueueTestServerFactory.ts | 48 ++++++++++++++++++ 17 files changed, 148 insertions(+), 178 deletions(-) create mode 100644 tests/queue/utils/QueueTestServerFactory.ts diff --git a/src/common/VSCServerManagerQueue.ts b/src/common/VSCServerManagerQueue.ts index 51005ef4c..2529ab7da 100644 --- a/src/common/VSCServerManagerQueue.ts +++ b/src/common/VSCServerManagerQueue.ts @@ -98,7 +98,8 @@ export default class VSCServerManagerBlob extends VSCServerManagerBase { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); return config; } diff --git a/src/common/VSCServerManagerTable.ts b/src/common/VSCServerManagerTable.ts index ec8c736e7..b830ab5c9 100644 --- a/src/common/VSCServerManagerTable.ts +++ b/src/common/VSCServerManagerTable.ts @@ -77,7 +77,8 @@ export default class VSCServerManagerTable extends VSCServerManagerBase { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); return config; } diff --git a/src/queue/handlers/MessageIdHandler.ts b/src/queue/handlers/MessageIdHandler.ts index d5f74ce90..f00c5b7f4 100644 --- a/src/queue/handlers/MessageIdHandler.ts +++ b/src/queue/handlers/MessageIdHandler.ts @@ -5,7 +5,7 @@ import Context from "../generated/Context"; import IMessageIdHandler from "../generated/handlers/IMessageIdHandler"; import { MessageUpdateProperties } from "../persistence/IQueueMetadataStore"; import { - DEFUALT_UPDATE_VISIBILITYTIMEOUT, + DEFAULT_UPDATE_VISIBILITYTIMEOUT, MESSAGETEXT_LENGTH_MAX, QUEUE_API_VERSION, UPDATE_VISIBILITYTIMEOUT_MAX, @@ -80,7 +80,7 @@ export default class MessageIdHandler extends BaseHandler // Validate the query parameters. const timeNextVisible = new Date( - context.startTime!.getTime() + DEFUALT_UPDATE_VISIBILITYTIMEOUT * 1000 // 30s as default + context.startTime!.getTime() + DEFAULT_UPDATE_VISIBILITYTIMEOUT * 1000 // 30s as default ); if (visibilitytimeout !== undefined) { if ( diff --git a/src/queue/handlers/MessagesHandler.ts b/src/queue/handlers/MessagesHandler.ts index c54755688..4be1192b7 100644 --- a/src/queue/handlers/MessagesHandler.ts +++ b/src/queue/handlers/MessagesHandler.ts @@ -7,8 +7,8 @@ import Context from "../generated/Context"; import IMessagesHandler from "../generated/handlers/IMessagesHandler"; import { MessageModel } from "../persistence/IQueueMetadataStore"; import { - DEFUALT_DEQUEUE_VISIBILITYTIMEOUT, - DEFUALT_MESSAGETTL, + DEFAULT_DEQUEUE_VISIBILITYTIMEOUT, + DEFAULT_MESSAGETTL, DEQUEUE_NUMOFMESSAGES_MAX, DEQUEUE_NUMOFMESSAGES_MIN, DEQUEUE_VISIBILITYTIMEOUT_MAX, @@ -58,7 +58,7 @@ export default class MessagesHandler extends BaseHandler // Validate the query parameters. const timeNextVisible = new Date( - context.startTime!.getTime() + DEFUALT_DEQUEUE_VISIBILITYTIMEOUT * 1000 // 30s as default, convert to ms + context.startTime!.getTime() + DEFAULT_DEQUEUE_VISIBILITYTIMEOUT * 1000 // 30s as default, convert to ms ); if (options.visibilitytimeout !== undefined) { if ( @@ -217,7 +217,7 @@ export default class MessagesHandler extends BaseHandler messageId: uuid(), insertionTime: new Date(context.startTime!), expirationTime: new Date( - context.startTime!.getTime() + DEFUALT_MESSAGETTL * 1000 + context.startTime!.getTime() + DEFAULT_MESSAGETTL * 1000 ), // Default ttl is 7 days. dequeueCount: 0, timeNextVisible: new Date(context.startTime!), diff --git a/src/queue/main.ts b/src/queue/main.ts index 534ddffb9..8f22dabbb 100644 --- a/src/queue/main.ts +++ b/src/queue/main.ts @@ -64,7 +64,8 @@ async function main() { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); // We use logger singleton as global debugger logger to track detailed outputs cross layers diff --git a/src/queue/utils/constants.ts b/src/queue/utils/constants.ts index 20cbf507c..4b4889a24 100644 --- a/src/queue/utils/constants.ts +++ b/src/queue/utils/constants.ts @@ -20,17 +20,17 @@ export const NEVER_EXPIRE_DATE = new Date("9999-12-31T23:59:59.999Z"); export const QUEUE_SERVICE_PERMISSION = "raup"; export const LIST_QUEUE_MAXRESSULTS_MIN = 1; export const LIST_QUEUE_MAXRESSULTS_MAX = 2147483647; -export const DEFUALT_DEQUEUE_VISIBILITYTIMEOUT = 30; // 30s as default. +export const DEFAULT_DEQUEUE_VISIBILITYTIMEOUT = 30; // 30s as default. export const DEQUEUE_VISIBILITYTIMEOUT_MIN = 1; export const DEQUEUE_VISIBILITYTIMEOUT_MAX = 604800; export const DEQUEUE_NUMOFMESSAGES_MIN = 1; export const DEQUEUE_NUMOFMESSAGES_MAX = 32; export const MESSAGETEXT_LENGTH_MAX = 65536; -export const DEFUALT_MESSAGETTL = 604800; // 604800s (7 day) as default. +export const DEFAULT_MESSAGETTL = 604800; // 604800s (7 day) as default. export const ENQUEUE_VISIBILITYTIMEOUT_MIN = 0; export const ENQUEUE_VISIBILITYTIMEOUT_MAX = 604800; export const MESSAGETTL_MIN = 1; -export const DEFUALT_UPDATE_VISIBILITYTIMEOUT = 30; // 30s as default. +export const DEFAULT_UPDATE_VISIBILITYTIMEOUT = 30; // 30s as default. export const UPDATE_VISIBILITYTIMEOUT_MIN = 0; export const UPDATE_VISIBILITYTIMEOUT_MAX = 604800; diff --git a/tests/queue/apis/messageid.test.ts b/tests/queue/apis/messageid.test.ts index 8827d86e4..095ea44f1 100644 --- a/tests/queue/apis/messageid.test.ts +++ b/tests/queue/apis/messageid.test.ts @@ -9,7 +9,6 @@ import { import { configLogger } from "../../../src/common/Logger"; import { StoreDestinationArray } from "../../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../../src/queue/QueueConfiguration"; import Server from "../../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, @@ -18,6 +17,7 @@ import { rmRecursive, sleep } from "../../testutils"; +import QueueTestServerFactory from "../utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -30,7 +30,7 @@ describe("MessageId APIs test", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -38,15 +38,6 @@ describe("MessageId APIs test", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - const baseURL = `http://${host}:${port}/devstoreaccount1`; const serviceClient = new QueueServiceClient( baseURL, @@ -68,7 +59,11 @@ describe("MessageId APIs test", () => { const messageContent = "Hello World"; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY, + }); await server.start(); }); diff --git a/tests/queue/apis/messages.test.ts b/tests/queue/apis/messages.test.ts index 2056491b6..01f961936 100644 --- a/tests/queue/apis/messages.test.ts +++ b/tests/queue/apis/messages.test.ts @@ -9,7 +9,6 @@ import { import { configLogger } from "../../../src/common/Logger"; import { StoreDestinationArray } from "../../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../../src/queue/QueueConfiguration"; import Server from "../../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, @@ -18,6 +17,7 @@ import { rmRecursive, sleep } from "../../testutils"; +import QueueTestServerFactory from "../utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -30,7 +30,7 @@ describe("Messages APIs test", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -38,15 +38,6 @@ describe("Messages APIs test", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - const baseURL = `http://${host}:${port}/devstoreaccount1`; const serviceClient = new QueueServiceClient( baseURL, @@ -67,7 +58,11 @@ describe("Messages APIs test", () => { const messageContent = "Hello World"; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }); await server.start(); }); diff --git a/tests/queue/apis/queue.test.ts b/tests/queue/apis/queue.test.ts index d5a1bf733..d1e1f247a 100644 --- a/tests/queue/apis/queue.test.ts +++ b/tests/queue/apis/queue.test.ts @@ -9,7 +9,6 @@ import { import { configLogger } from "../../../src/common/Logger"; import { StoreDestinationArray } from "../../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../../src/queue/QueueConfiguration"; import Server from "../../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, @@ -17,6 +16,7 @@ import { getUniqueName, rmRecursive } from "../../testutils"; +import QueueTestServerFactory from "../utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -29,7 +29,7 @@ describe("Queue APIs test", () => { const extentDbPath = "__queueExtentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -37,15 +37,6 @@ describe("Queue APIs test", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - const baseURL = `http://${host}:${port}/devstoreaccount1`; const serviceClient = new QueueServiceClient( baseURL, @@ -65,7 +56,11 @@ describe("Queue APIs test", () => { let queueClient: QueueClient; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }); await server.start(); }); diff --git a/tests/queue/apis/queueService.test.ts b/tests/queue/apis/queueService.test.ts index ff5de1e52..519282e6c 100644 --- a/tests/queue/apis/queueService.test.ts +++ b/tests/queue/apis/queueService.test.ts @@ -7,7 +7,6 @@ import * as assert from "assert"; import { configLogger } from "../../../src/common/Logger"; import { StoreDestinationArray } from "../../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../../src/queue/QueueConfiguration"; import Server from "../../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, @@ -16,6 +15,7 @@ import { rmRecursive, sleep } from "../../testutils"; +import QueueTestServerFactory from "../utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -28,7 +28,7 @@ describe("QueueServiceAPIs", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -36,15 +36,6 @@ describe("QueueServiceAPIs", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - const baseURL = `http://${host}:${port}/devstoreaccount1`; const serviceClient = new QueueServiceClient( baseURL, @@ -62,7 +53,11 @@ describe("QueueServiceAPIs", () => { let server: Server; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }) await server.start(); }); @@ -270,7 +265,7 @@ describe("QueueServiceAPIs - secondary location endpoint", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -278,15 +273,6 @@ describe("QueueServiceAPIs - secondary location endpoint", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - const baseURL = `http://${host}:${port}/devstoreaccount1-secondary`; const serviceClient = new QueueServiceClient( baseURL, @@ -304,7 +290,11 @@ describe("QueueServiceAPIs - secondary location endpoint", () => { let server: Server; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }); await server.start(); }); diff --git a/tests/queue/https.test.ts b/tests/queue/https.test.ts index d342996b5..97807dbb7 100644 --- a/tests/queue/https.test.ts +++ b/tests/queue/https.test.ts @@ -6,7 +6,6 @@ import { import { configLogger } from "../../src/common/Logger"; import { StoreDestinationArray } from "../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../src/queue/QueueConfiguration"; import Server from "../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, @@ -14,6 +13,7 @@ import { getUniqueName, rmRecursive } from "../testutils"; +import QueueTestServerFactory from "./utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -26,7 +26,7 @@ describe("Queue HTTPS", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -34,26 +34,15 @@ describe("Queue HTTPS", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false, - undefined, - undefined, - undefined, - false, - false, - "tests/server.cert", - "tests/server.key" - ); - let server: Server; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY, + https: true + }); await server.start(); }); diff --git a/tests/queue/oauth.test.ts b/tests/queue/oauth.test.ts index 7494e0b37..99f394181 100644 --- a/tests/queue/oauth.test.ts +++ b/tests/queue/oauth.test.ts @@ -5,9 +5,9 @@ import Server from "../../src/queue/QueueServer"; import { configLogger } from "../../src/common/Logger"; import { StoreDestinationArray } from "../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../src/queue/QueueConfiguration"; import { EMULATOR_ACCOUNT_KEY, generateJWTToken, getUniqueName } from "../testutils"; import { SimpleTokenCredential } from "../simpleTokenCredential"; +import QueueTestServerFactory from "./utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -20,7 +20,7 @@ describe("Queue OAuth Basic", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -28,30 +28,18 @@ describe("Queue OAuth Basic", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false, - undefined, - undefined, - undefined, - false, - false, - "tests/server.cert", - "tests/server.key", - undefined, - "basic" - ); - let server: Server; const baseURL = `https://${host}:${port}/devstoreaccount1`; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY, + https: true, + oauth: "basic" + }) await server.start(); }); @@ -534,25 +522,12 @@ describe("Queue OAuth Basic", () => { await server.close(); await server.clean(); - server = new Server( - new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false, - undefined, - undefined, - undefined, - false, - false, - undefined, - undefined, - undefined, - "basic" - ) - ); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY, + oauth: "basic" + }) await server.start(); const httpBaseURL = `http://${server.config.host}:${server.config.port}/devstoreaccount1`; diff --git a/tests/queue/queueAuthentication.test.ts b/tests/queue/queueAuthentication.test.ts index 2994b7912..0d4f72b32 100644 --- a/tests/queue/queueAuthentication.test.ts +++ b/tests/queue/queueAuthentication.test.ts @@ -9,7 +9,6 @@ import { import { configLogger } from "../../src/common/Logger"; import { StoreDestinationArray } from "../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../src/queue/QueueConfiguration"; import Server from "../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, @@ -17,6 +16,7 @@ import { getUniqueName, rmRecursive } from "../testutils"; +import QueueTestServerFactory from "./utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -29,7 +29,7 @@ describe("Queue Authentication", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -37,19 +37,14 @@ describe("Queue Authentication", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - let server: Server; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }); await server.start(); }); diff --git a/tests/queue/queueCorsRequest.test.ts b/tests/queue/queueCorsRequest.test.ts index 9e06d97f5..5753b2681 100644 --- a/tests/queue/queueCorsRequest.test.ts +++ b/tests/queue/queueCorsRequest.test.ts @@ -7,7 +7,6 @@ import * as assert from "assert"; import { configLogger } from "../../src/common/Logger"; import { StoreDestinationArray } from "../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../src/queue/QueueConfiguration"; import Server from "../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, @@ -17,6 +16,7 @@ import { } from "../testutils"; import OPTIONSRequestPolicyFactory from "./RequestPolicy/OPTIONSRequestPolicyFactory"; import OriginPolicyFactory from "./RequestPolicy/OriginPolicyFactory"; +import QueueTestServerFactory from "./utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -29,7 +29,7 @@ describe("Queue Cors requests test", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -37,15 +37,6 @@ describe("Queue Cors requests test", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - const baseURL = `http://${host}:${port}/devstoreaccount1`; const serviceClient = new QueueServiceClient( baseURL, @@ -62,7 +53,11 @@ describe("Queue Cors requests test", () => { let server: Server; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: metadataDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }); await server.start(); }); diff --git a/tests/queue/queueSas.test.ts b/tests/queue/queueSas.test.ts index 05f641f5a..e917434bc 100644 --- a/tests/queue/queueSas.test.ts +++ b/tests/queue/queueSas.test.ts @@ -17,7 +17,6 @@ import { import { configLogger } from "../../src/common/Logger"; import { StoreDestinationArray } from "../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../src/queue/QueueConfiguration"; import Server from "../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, @@ -26,6 +25,7 @@ import { rmRecursive, sleep } from "../testutils"; +import QueueTestServerFactory from "./utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -38,7 +38,7 @@ describe("Queue SAS test", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -46,15 +46,6 @@ describe("Queue SAS test", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - const baseURL = `http://${host}:${port}/devstoreaccount1`; const serviceClient = new QueueServiceClient( baseURL, @@ -72,7 +63,11 @@ describe("Queue SAS test", () => { let server: Server; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }); await server.start(); }); diff --git a/tests/queue/queueSpecialnaming.test.ts b/tests/queue/queueSpecialnaming.test.ts index e870059d5..735ecbdcf 100644 --- a/tests/queue/queueSpecialnaming.test.ts +++ b/tests/queue/queueSpecialnaming.test.ts @@ -8,13 +8,13 @@ import { import { configLogger } from "../../src/common/Logger"; import { StoreDestinationArray } from "../../src/common/persistence/IExtentStore"; -import QueueConfiguration from "../../src/queue/QueueConfiguration"; import Server from "../../src/queue/QueueServer"; import { EMULATOR_ACCOUNT_KEY, EMULATOR_ACCOUNT_NAME, rmRecursive } from "../testutils"; +import QueueTestServerFactory from "./utils/QueueTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -27,7 +27,7 @@ describe("Queue SpecialNaming", () => { const extentDbPath = "__extentTestsStorage__"; const persistencePath = "__queueTestsPersistence__"; - const DEFUALT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ + const DEFAULT_QUEUE_PERSISTENCE_ARRAY: StoreDestinationArray = [ { locationId: "queueTest", locationPath: persistencePath, @@ -35,15 +35,6 @@ describe("Queue SpecialNaming", () => { } ]; - const config = new QueueConfiguration( - host, - port, - metadataDbPath, - extentDbPath, - DEFUALT_QUEUE_PERSISTENCE_ARRAY, - false - ); - const baseURL = `http://${host}:${port}/devstoreaccount1`; const serviceClient = new QueueServiceClient( baseURL, @@ -61,7 +52,11 @@ describe("Queue SpecialNaming", () => { let server: Server; before(async () => { - server = new Server(config); + server = new QueueTestServerFactory().createServer({ + metadataDBPath: metadataDbPath, + extentDBPath: extentDbPath, + persistencePathArray: DEFAULT_QUEUE_PERSISTENCE_ARRAY + }); await server.start(); }); diff --git a/tests/queue/utils/QueueTestServerFactory.ts b/tests/queue/utils/QueueTestServerFactory.ts new file mode 100644 index 000000000..de2c0303a --- /dev/null +++ b/tests/queue/utils/QueueTestServerFactory.ts @@ -0,0 +1,48 @@ +import { StoreDestinationArray } from "../../../src/common/persistence/IExtentStore" +import QueueConfiguration from "../../../src/queue/QueueConfiguration" +import QueueServer from "../../../src/queue/QueueServer" + +export interface IQueueTestServerFactoryParams { + metadataDBPath: string + extentDBPath: string + persistencePathArray: StoreDestinationArray + enableDebugLog?: boolean + debugLogFilePath?: string + loose?: boolean + skipApiVersionCheck?: boolean + https?: boolean + oauth?: string +} + +export default class QueueTestServerFactory { + public createServer(params: IQueueTestServerFactoryParams): QueueServer { + const inMemoryPersistence = process.env.AZURITE_TEST_INMEMORYPERSISTENCE !== undefined; + + const port = 11001; + const host = "127.0.0.1"; + + const cert = params.https ? "tests/server.cert" : undefined; + const key = params.https ? "tests/server.key" : undefined; + + const config = new QueueConfiguration( + host, + port, + params.metadataDBPath, + params.extentDBPath, + params.persistencePathArray, + false, + undefined, + params.enableDebugLog, + params.debugLogFilePath, + params.loose, + params.skipApiVersionCheck, + cert, + key, + undefined, + params.oauth, + undefined, + inMemoryPersistence + ); + return new QueueServer(config); + } +} From c3aee9122636401186b30b6011d92a84071f56eb Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 02:17:28 -0400 Subject: [PATCH 10/37] Properly handle offset and count in extents Fix tests --- .vscode/launch.json | 28 ++++++++++++++ src/common/persistence/MemoryExtentStore.ts | 41 ++++++++++++++++++--- tests/table/apis/table.entity.query.test.ts | 4 +- tests/table/utils/TableTestServerFactory.ts | 6 ++- 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 0aa75a80a..02d73d813 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -217,6 +217,34 @@ "internalConsoleOptions": "openOnSessionStart", "outputCapture": "std" }, + { + "type": "node", + "request": "launch", + "name": "Current Mocha TS File - Loki, in-memory", + "cwd": "${workspaceFolder}", + "runtimeArgs": ["-r", "ts-node/register"], + "args": [ + "${workspaceFolder}/node_modules/mocha/bin/_mocha", + "-u", + "tdd", + "--timeout", + "999999", + "--colors", + "${workspaceFolder}/${relativeFile}" + ], + "env": { + "AZURITE_ACCOUNTS": "", + "AZURE_TABLE_STORAGE": "", + "DATATABLES_ACCOUNT_NAME": "", + "DATATABLES_ACCOUNT_KEY" : "", + "AZURE_DATATABLES_STORAGE_STRING": "https://.table.core.windows.net", + "AZURE_DATATABLES_SAS": "?", + "NODE_TLS_REJECT_UNAUTHORIZED": "0", + "AZURITE_TEST_INMEMORYPERSISTENCE": "true" + }, + "internalConsoleOptions": "openOnSessionStart", + "outputCapture": "std" + }, { "name": "VSC Extension", "type": "extensionHost", diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts index 2ecab34d0..665381fb5 100644 --- a/src/common/persistence/MemoryExtentStore.ts +++ b/src/common/persistence/MemoryExtentStore.ts @@ -64,12 +64,16 @@ export default class MemoryExtentStore implements IExtentStore { const chunks: (Buffer | string)[] = [] let count = 0; if (data instanceof Buffer) { - chunks.push(data) - count = data.length + if (data.length > 0) { + chunks.push(data) + count = data.length + } } else { for await (let chunk of data) { - chunks.push(chunk) - count += chunk.length + if (chunk.length > 0) { + chunks.push(chunk) + count += chunk.length + } } } @@ -111,7 +115,34 @@ export default class MemoryExtentStore implements IExtentStore { } const buffer = new Readable() - match.chunks.forEach(chunk => buffer.push(chunk)) + let skip = extentChunk.offset; + let take = extentChunk.count; + for (const chunk of match.chunks) { + if (skip > 0) { + if (chunk.length <= skip) { + // this chunk is entirely skipped + skip -= chunk.length + } else { + // part of the chunk is included + const end = skip + Math.min(take, chunk.length - skip) + const slice = chunk.slice(skip, end); + buffer.push(chunk.slice(skip, end)) + skip = 0 + take -= slice.length + } + } else { + if (chunk.length > take) { + // all of the chunk is included, up to the count limit + const slice = chunk.slice(0, take); + buffer.push(slice) + take -= slice.length + } else { + // all of the chunk is included + buffer.push(chunk) + take -= chunk.length + } + } + } buffer.push(null) return buffer; diff --git a/tests/table/apis/table.entity.query.test.ts b/tests/table/apis/table.entity.query.test.ts index 746c6163d..e89ec2c9b 100644 --- a/tests/table/apis/table.entity.query.test.ts +++ b/tests/table/apis/table.entity.query.test.ts @@ -15,6 +15,7 @@ import { createUniquePartitionKey } from "../utils/table.entity.test.utils"; import uuid from "uuid"; +import TableTestServerFactory from "../utils/TableTestServerFactory"; // import uuid from "uuid"; // Set true to enable debug log configLogger(false); @@ -798,7 +799,8 @@ describe("table Entity APIs test - using Azure/data-tables", () => { await tableClient.deleteTable(); }); - it("13. should find both old and new guids (backwards compatible) when using guid type, @loki", async () => { + // Skip the case when running in-memory. Backwards compatibility with old DBs does not apply. + (TableTestServerFactory.inMemoryPersistence() ? it.skip : it)("13. should find both old and new guids (backwards compatible) when using guid type, @loki", async () => { const tableClient = createAzureDataTablesClient( testLocalAzuriteInstance, "reproTable" diff --git a/tests/table/utils/TableTestServerFactory.ts b/tests/table/utils/TableTestServerFactory.ts index 7b97197ee..8f7ace176 100644 --- a/tests/table/utils/TableTestServerFactory.ts +++ b/tests/table/utils/TableTestServerFactory.ts @@ -12,8 +12,12 @@ export interface ITableTestServerFactoryParams { } export default class TableTestServerFactory { + public static inMemoryPersistence() { + return process.env.AZURITE_TEST_INMEMORYPERSISTENCE !== undefined; + } + public createServer(params: ITableTestServerFactoryParams): TableServer { - const inMemoryPersistence = process.env.AZURITE_TEST_INMEMORYPERSISTENCE !== undefined; + const inMemoryPersistence = TableTestServerFactory.inMemoryPersistence() const port = 11002; const host = "127.0.0.1"; From 939291a225897f995c36dfbf89f7d890e16dda71 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 02:23:50 -0400 Subject: [PATCH 11/37] Add in-memory tests to all variants --- azure-pipelines.yml | 68 ++++++++++++++++++++++++++++++++++++++++++--- package.json | 4 +++ 2 files changed, 68 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0b7fb01b0..598f86734 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,7 +37,13 @@ jobs: workingDirectory: "./" displayName: "npm run test:blob" env: {} - + + - script: | + npm run test:blob:in-memory + workingDirectory: "./" + displayName: "npm run test:blob:in-memory" + env: {} + - job: blobtestubuntu22_04 displayName: Blob Test Linux Ubuntu 22.04 pool: @@ -53,18 +59,24 @@ jobs: inputs: versionSpec: "$(node_version)" displayName: "Install Node.js" - + - script: | npm ci --legacy-peer-deps workingDirectory: "./" displayName: "npm ci --legacy-peer-deps" - + - script: | npm run test:blob workingDirectory: "./" displayName: "npm run test:blob" env: {} + - script: | + npm run test:blob:in-memory + workingDirectory: "./" + displayName: "npm run test:blob:in-memory" + env: {} + - job: blobtestwin displayName: Blob Test Windows pool: @@ -92,6 +104,12 @@ jobs: displayName: "npm run test:blob" env: {} + - script: | + npm run test:blob:in-memory + workingDirectory: "./" + displayName: "npm run test:blob:in-memory" + env: {} + - job: blobtestmac displayName: Blob Test Mac pool: @@ -119,6 +137,12 @@ jobs: displayName: "npm run test:blob" env: {} + - script: | + npm run test:blob:in-memory + workingDirectory: "./" + displayName: "npm run test:blob:in-memory" + env: {} + - job: blobtestmysql displayName: Blob Test Mysql pool: @@ -181,6 +205,12 @@ jobs: displayName: "npm run test:queue" env: {} + - script: | + npm run test:queue:in-memory + workingDirectory: "./" + displayName: "npm run test:queue:in-memory" + env: {} + - job: queuetestwin displayName: Queue Test Windows pool: @@ -208,6 +238,12 @@ jobs: displayName: "npm run test:queue" env: {} + - script: | + npm run test:queue:in-memory + workingDirectory: "./" + displayName: "npm run test:queue:in-memory" + env: {} + - job: queuetestmac displayName: Queue Test Mac pool: @@ -235,6 +271,12 @@ jobs: displayName: "npm run test:queue" env: {} + - script: | + npm run test:queue:in-memory + workingDirectory: "./" + displayName: "npm run test:queue:in-memory" + env: {} + - job: tabletestlinux displayName: Table Test Linux pool: @@ -263,6 +305,12 @@ jobs: displayName: "npm run test:table" env: {} + - script: | + npm run test:table:in-memory + workingDirectory: "./" + displayName: "npm run test:table:in-memory" + env: {} + - job: tabletestwin displayName: Table Test Windows pool: @@ -291,6 +339,12 @@ jobs: displayName: "npm run test:table" env: {} + - script: | + npm run test:table:in-memory + workingDirectory: "./" + displayName: "npm run test:table:in-memory" + env: {} + - job: tabletestmac displayName: Table Test Mac pool: @@ -319,6 +373,12 @@ jobs: displayName: "npm run test:table" env: {} + - script: | + npm run test:table:in-memory + workingDirectory: "./" + displayName: "npm run test:table:in-memory" + env: {} + - job: azuritenodejslinux displayName: Azurite Linux pool: @@ -326,7 +386,7 @@ jobs: strategy: matrix: # Table tests no longer suport older node versions - # skip node 14 Azurite install test, since it has issue iwth new npm, which is not azurite issue. + # skip node 14 Azurite install test, since it has issue iwth new npm, which is not azurite issue. # Track with https://github.com/Azure/Azurite/issues/1550. Will add node 14 back later when the issue resolved. #node_14_x: # node_version: 14.x diff --git a/package.json b/package.json index 952136c2f..364296298 100644 --- a/package.json +++ b/package.json @@ -286,11 +286,15 @@ "azurite": "node -r ts-node/register src/azurite.ts", "lint": "npx eslint src/**/*.ts", "test": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --grep @loki --recursive --exit tests/**/*.test.ts tests/**/**/*.test.ts", + "test:in-memory": "npm run lint && cross-env AZURITE_TEST_INMEMORYPERSISTENCE=1 NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --grep @loki --recursive --exit tests/**/*.test.ts tests/**/**/*.test.ts", "test:blob": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --grep @loki --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts", + "test:blob:in-memory": "npm run lint && cross-env AZURITE_TEST_INMEMORYPERSISTENCE=1 NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --grep @loki --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts", "test:blob:sql": "npm run lint && cross-env cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 AZURITE_TEST_DB=mysql://root:my-secret-pw@127.0.0.1:3306/azurite_blob_test mocha --compilers ts-node/register --no-timeouts --grep @sql --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts", "test:blob:sql:ci": "npm run lint && cross-env cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 AZURITE_TEST_DB=mysql://root:my-secret-pw@127.0.0.1:13306/azurite_blob_test mocha --compilers ts-node/register --no-timeouts --grep @sql --recursive --exit tests/blob/*.test.ts tests/blob/**/*.test.ts", "test:queue": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --recursive --exit tests/queue/*.test.ts tests/queue/**/*.test.ts", + "test:queue:in-memory": "npm run lint && cross-env AZURITE_TEST_INMEMORYPERSISTENCE=1 NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --recursive --exit tests/queue/*.test.ts tests/queue/**/*.test.ts", "test:table": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --recursive --exit tests/table/**/*.test.ts", + "test:table:in-memory": "npm run lint && cross-env AZURITE_TEST_INMEMORYPERSISTENCE=1 NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts --recursive --exit tests/table/**/*.test.ts", "test:exe": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts tests/exe.test.ts --exit", "test:linux": "npm run lint && cross-env NODE_TLS_REJECT_UNAUTHORIZED=0 mocha --compilers ts-node/register --no-timeouts tests/linuxbinary.test.ts --exit", "clean": "rimraf dist typings *.log coverage __testspersistence__ temp __testsstorage__ .nyc_output debug.log *.vsix *.tgz", From 2ae4746afc01826d364643f09c66b130d0efd404 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 10:05:21 -0400 Subject: [PATCH 12/37] Fix two flaky tests --- .../table/apis/table.validation.rest.test.ts | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/tests/table/apis/table.validation.rest.test.ts b/tests/table/apis/table.validation.rest.test.ts index 7199382c9..b6cf46a2e 100644 --- a/tests/table/apis/table.validation.rest.test.ts +++ b/tests/table/apis/table.validation.rest.test.ts @@ -188,8 +188,12 @@ describe("table name validation tests", () => { }); it("should create a table with a name which is a substring of an existing table, @loki", async () => { - tableName = getUniqueName("table"); - const body = JSON.stringify({ + // this will be used for 3 table names + // [ a + b ], [ a ], [ b ] + const a = getUniqueName("t") + const b = getUniqueName("t") + tableName = a + b; + const body1 = JSON.stringify({ TableName: tableName }); const createTableHeaders = { @@ -197,7 +201,7 @@ describe("table name validation tests", () => { Accept: "application/json;odata=nometadata" }; try { - await postToAzurite("Tables", body, createTableHeaders); + await postToAzurite("Tables", body1, createTableHeaders); } catch (err: any) { assert.strictEqual( err.response.status, @@ -205,9 +209,8 @@ describe("table name validation tests", () => { `unexpected status code : ${err.response.status}` ); } - const tableName2 = tableName.substring(0, tableName.length - 4); const body2 = JSON.stringify({ - TableName: tableName2 + TableName: a }); try { await postToAzurite("Tables", body2, createTableHeaders); @@ -218,9 +221,8 @@ describe("table name validation tests", () => { `unexpected exception with end trimmed! : ${err}` ); } - const tableName3 = tableName.substring(4); const body3 = JSON.stringify({ - TableName: tableName3 + TableName: b }); try { await postToAzurite("Tables", body3, createTableHeaders); @@ -234,10 +236,14 @@ describe("table name validation tests", () => { }); it("should delete a table with a name which is a substring of an existing table, @loki", async () => { - tableName = getUniqueName("table"); - const shortName = tableName.substring(0, 6); - const basicBody = JSON.stringify({ - TableName: shortName + // this will be used for 4 table names + // [ a ], [ a + b + c ], [ a + b ], [ b + c ] + const a = getUniqueName("t") + const b = getUniqueName("t") + const c = getUniqueName("t") + + const body1 = JSON.stringify({ + TableName: a }); const createTableHeaders = { "Content-Type": "application/json", @@ -251,7 +257,7 @@ describe("table name validation tests", () => { try { const createTable1 = await postToAzurite( "Tables", - basicBody, + body1, createTableHeaders ); assert.strictEqual( @@ -266,14 +272,14 @@ describe("table name validation tests", () => { `unexpected status code creating first table : ${err.response.status}` ); } - const body = JSON.stringify({ - TableName: tableName + const body2 = JSON.stringify({ + TableName: a + b + c }); // create table 2 try { const createTable2 = await postToAzurite( "Tables", - body, + body2, createTableHeaders ); assert.strictEqual( @@ -288,15 +294,14 @@ describe("table name validation tests", () => { `unexpected status code : ${err.response.status}` ); } - const tableName2 = tableName.substring(0, tableName.length - 4); - const body2 = JSON.stringify({ - TableName: tableName2 + const body3 = JSON.stringify({ + TableName: a + b }); // create table 3 try { const createTable3 = await postToAzurite( "Tables", - body2, + body3, createTableHeaders ); assert.strictEqual( @@ -311,15 +316,14 @@ describe("table name validation tests", () => { `unexpected exception with end trimmed! : ${err}` ); } - const tableName3 = tableName.substring(4); - const body3 = JSON.stringify({ - TableName: tableName3 + const body4 = JSON.stringify({ + TableName: b + c }); // create table 4 try { const createTable4 = await postToAzurite( "Tables", - body3, + body4, createTableHeaders ); assert.strictEqual( @@ -334,7 +338,7 @@ describe("table name validation tests", () => { `unexpected exception with start trimmed! : ${err}` ); } - // now list tables after deletion... + // now list tables before deletion... try { const listTableResult1 = await getToAzurite("Tables", createTableHeaders); // we count all tables created in the tests before this one as well @@ -353,7 +357,7 @@ describe("table name validation tests", () => { // now delete "table" try { const deleteResult = await deleteToAzurite( - `Tables('${shortName}')`, + `Tables('${a}')`, "", createTableHeaders ); @@ -380,7 +384,7 @@ describe("table name validation tests", () => { for (const table of listTableResult2.data.value) { assert.notStrictEqual( table.TableName, - shortName, + a, "We still list the table we should have deleted." ); } From b006592f3b7c9e3f9404cc12702c71ac447c018c Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 10:05:56 -0400 Subject: [PATCH 13/37] Make random local files have some newline characters so diff tools are usable --- tests/blob/blockblob.highlevel.test.ts | 8 ++++---- tests/testutils.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/blob/blockblob.highlevel.test.ts b/tests/blob/blockblob.highlevel.test.ts index 22bac9fb3..f2057eeed 100644 --- a/tests/blob/blockblob.highlevel.test.ts +++ b/tests/blob/blockblob.highlevel.test.ts @@ -78,14 +78,14 @@ describe("BlockBlobHighlevel", () => { } tempFileLarge = await createRandomLocalFile( tempFolderPath, - 257, - 1024 * 1024 + 8224, + 1024 * 32 ); tempFileLargeLength = 257 * 1024 * 1024; tempFileSmall = await createRandomLocalFile( tempFolderPath, - 15, - 1024 * 1024 + 480, + 1024 * 32 ); tempFileSmallLength = 15 * 1024 * 1024; }); diff --git a/tests/testutils.ts b/tests/testutils.ts index 560d7b39a..86c214622 100644 --- a/tests/testutils.ts +++ b/tests/testutils.ts @@ -140,7 +140,7 @@ export async function createRandomLocalFile( function randomValueHex(len = blockSize) { return randomBytes(Math.ceil(len / 2)) .toString("hex") // convert to hexadecimal format - .slice(0, len); // return required number of characters + .slice(0, len - (len > 1 ? 1 : 0)) + (len > 1 ? "\n" : ""); // append newlines to make debugging easier } ws.on("open", () => { From ba1410b5656e5510ac10d123edc1abfffd2ea74d Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 10:06:20 -0400 Subject: [PATCH 14/37] Use a larger buffer for the download, the test is very slow otherwise --- tests/blob/blockblob.highlevel.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/blob/blockblob.highlevel.test.ts b/tests/blob/blockblob.highlevel.test.ts index f2057eeed..999b8df68 100644 --- a/tests/blob/blockblob.highlevel.test.ts +++ b/tests/blob/blockblob.highlevel.test.ts @@ -306,7 +306,7 @@ describe("BlockBlobHighlevel", () => { const aborter = new AbortController(); try { await blockBlobClient.downloadToBuffer(buf, 0, undefined, { - blockSize: 1 * 1024, + blockSize: 32 * 1024, // if too small, the test is very slow maxRetryRequestsPerBlock: 5, concurrency: 1, onProgress: () => { From 7d2bf7fcd755a5c6a9dfc216cfb4e7ee40ec3bef Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 13:26:50 -0400 Subject: [PATCH 15/37] Add design spec --- docs/designs/2023-10-in-memory-persistence.md | 171 ++++++++++++++++++ .../high-memory.png | Bin 0 -> 106165 bytes 2 files changed, 171 insertions(+) create mode 100644 docs/designs/2023-10-in-memory-persistence.md create mode 100644 docs/designs/meta/resources/2023-10-in-memory-persistence/high-memory.png diff --git a/docs/designs/2023-10-in-memory-persistence.md b/docs/designs/2023-10-in-memory-persistence.md new file mode 100644 index 000000000..1565ce2ee --- /dev/null +++ b/docs/designs/2023-10-in-memory-persistence.md @@ -0,0 +1,171 @@ +# Add an option to use only in-memory persistence + +- Author Name: Joel Verhagen ([@joelverhagen](https://github.com/joelverhagen)) +- GitHub Issue: [Azure/Azurite#2227](https://github.com/Azure/Azurite/issues/2227) + +## Summary + +Currently, Azurite uses disk-based persistence for all three storage endpoints (blobs, queues, tables). It is not +possible to change this default behavior to store information only in-memory. This proposal suggests the addition of an +`--inMemoryPersistence` option for Azurite which will disable all disk-based persistence and instead store the data only +in-memory of the Azurite process (node). + +## Motivation + +The current usage pattern of Azurite assumes that the caller is interested in persisting stored data beyond the lifetime +of the node process. This certainly is useful for extended testing scenarios or where Azurite is actually used for +production-like scenarios (perhaps a best-effort approach of free, on-prem Azure Storage). However, some Azurite use +cases are more ephemeral in nature. For example, if you simply want to use Azurite for a single test run in a CI +pipeline, you don't need the storage to persist beyond the lifetime of the node process. + +This sample owned by the Azure-Samples GitHub organization illustrates such a case: + +[`automated-testing-with-azurite/build/azure-pipelines.yml`](https://github.com/Azure-Samples/automated-testing-with-azurite/blob/fb5323344056cfb2e43cbff91ed53858a71b2d8c/build/azure-pipelines.yml#L14-L22) + +For the NuGet team's [NuGet/Insights](https://github.com/NuGet/Insights) project, a similar approach is used in an Azure +DevOps release pipeline, allowing a simple, no-frills Azure Storage instance available with a simple `npx azurite`. + +[`Insights/.pipelines/Retail.PullRequest.yml`](https://github.com/NuGet/Insights/blob/5d0b5ed62a71b10d93d77f5135b4876a1b1bf4dc/.pipelines/Retail.PullRequest.yml#L164-L189) + +Without the option to disable disk-based persistence, an ephemeral use of Azurite requires an explicit clean-up step by +some actor to ensure disk space is not filled up over time. In the case of cloud-hosted Azure DevOps agents, this is +sometimes handled by the build agent software (cleaning up the working directory automatically, for example) but I think +it's best not to assume this, especially since it is possible that the write location of the Azurite data is +inadvertently outside of the scope of any auto-cleanup process. + +The disk-based persistence also incurs an additional performance penalty, especially when writing extents. This is +particularly egregious for queue messages that have a blocking disk operation per queue message enqueue and dequeue. In +some situations this performance is tolerable but for extended integration tests with dozens or even hundreds of queue +messages processed, the performance delta is noticeable. In a prototype implementation of this design, the following +performance improvement for blob and queue was seen for a simple write-then-read pattern for each storage service. + +| Method | Mean | Error | StdDev | +| -------------- | ---------: | --------: | --------: | +| InMemory_Blob | 1,860.0 us | 25.45 us | 21.25 us | +| InMemory_Queue | 1,562.3 us | 26.89 us | 25.15 us | +| InMemory_Table | 975.7 us | 8.65 us | 7.23 us | +| Disk_Blob | 5,505.8 us | 91.37 us | 89.74 us | +| Disk_Queue | 5,327.1 us | 104.47 us | 111.78 us | +| Disk_Table | 963.6 us | 11.34 us | 9.47 us | + +As you can see, the sample blob and queue workflow (write then read) is nearly 3 times faster without the disk +operation. Table storage operations have no significant change because the 5-second disk persistence step done by LokiJS +is not easily captured in a micro-benchmark. + +([benchmark implementation available on Gist](https://gist.github.com/joelverhagen/8394b3eaa276f4baa07806867b5caa37)) + +For the current disk-based implementation, there are no strong consistency or checkpoint guarantees. The LokiJS +persistence configuration has auto-saving set to true, but this is only done every 5 seconds meaning an abrupt +termination of the node process can still lead to data loss (much like an in-memory-only persistence model suggested by +this design). + +Finally, additional failure modes are present when disk-based persistence is enabled. Several issues have been opened in +the past related to this. For users where long-lived storage is needed, this design will not solve their problems since +they need the disk storage. For other users where ephemeral storage is acceptable if not ideal, this design will resolve +those problems. Example issues: + +- [Azure/Azurite#1687](https://github.com/Azure/Azurite/issues/1687), [Azure/Azurite#804 + (comment)](https://github.com/Azure/Azurite/issues/803#issuecomment-1244987098), - additional permissions may be + needed for disk persistence, depending on the configured path +- [Azure/Azurite#1967](https://github.com/Azure/Azurite/issues/1967) - there are OS limits for file handles + +## Explanation + +### Functional explanation + +A ``--inMemoryPersistence`` command line option will be introduced which will do two things: + +1. Switch all LokiJS instances from the default `fs` (file system) persistence model to `memory`. +1. Use a `MemoryExtentStore` for blob and queue extent storage instead of `FSExtentStore`. + +The SQL-based persistence model will support the ``--inMemoryPersistence`` option however this particular use-case is +already strange because some storage for Azurite is on a potentially remote SQL instance and other storage is on the +Azurite local file system. This would simply allow the extent storage to instead be on the Azurite local memory. + +Similar options will be added to the various related configuration types, configuration providers, and the VS Code +options. + +In all cases, the ``--inMemoryPersistence`` option will default to "off" retaining existing behavior by default. This +ensures backward compatibility but will provide the ability for users to cautiously opt-in to this behavior. + +### Technical explanation + +The LokiJS module already supports in-memory persistence out of the box. This can be opted into by using the following +options parameter to the constructor: +```typescript +{ + persistenceMethod: "memory" +} +``` + +Note that LokiJS already is primarily an in-memory store, but Azurite's current usage of it periodically writes the data +to disk to best-effort persistence spanning multiple process lifetimes. + +A new `MemoryExtentStore` implementation will be introduced to be used instead of `FSExtentStore`. This will store +extents in an instance-level map. The key of the map will be a UUID (GUID) and the value will be an extension of the +existing `IExtentChunk` interface. The extension will be `IMemoryExtentChunk` which has a single additional property: +```typescript +export interface IMemoryExtentChunk extends IExtentChunk { + chunks: (Buffer | string)[] +} +``` + +This will allow extent write operations to be stored in-memory (a list of buffered chunks) and read operations to fetch +the list of chunks on demand. The implementation will be simpler than the `FSExtentStore` in that all extent chunks will +have their own extent ID instead of sometimes being appended to existing extent chunks. + +All unit tests will be run on both the existing disk-based persistence model and the in-memory model. There is one +exception which is the `13. should find both old and new guids (backward compatible) when using guid type, @loki` test +case. This operates on a legacy LokiJS disk-based DB which is a scenario that is not applicable to an in-memory model +since old legacy DBs cannot be used. + +The test cases will use the in-memory-based persistence when the `AZURITE_TEST_INMEMORYPERSISTENCE` environment variable +is set to a truthy value. + +## Drawbacks + +Because extents are no longer stored on disk, the limit on the amount of data Azurite can store in this mode is no +longer bounded by the available disk space but is instead bounded by the amount of memory available to the node process. + +As a side note, the limit on the memory available to the node process is only loosely related to the physical memory of +the host machine. Windows, Linux, and macOS all support virtual memory which allows blocks of memory to be paged onto +disk. + +As you can see, a load test of the in-memory implementation allows uploads exceeding 64 GiB (which was the amount of +physical memory available on the test machine). There will be a noticeable performance drop when this happens but it +illustrates how the storage available to `--inMemoryPersistence` can be quite large. + +![high memory](meta/resources/2023-10-in-memory-persistence/high-memory.png) + +([load test implementation available on Gist](https://gist.github.com/joelverhagen/8394b3eaa276f4baa07806867b5caa37)) + +This can be a problem if Azurite is used for persisting very large files or many thousands of large queue messages. + +If the user of Azurite needs to store large files, then they should avoid using this feature and instead rely on +disk-based persistence. + +## Rationale and alternatives + +N/A + +## Prior Art + +N/A + +## Unresolved Questions + +N/A + +## Future Possibilities + +We could consider a more complex set of configuration values where the user can decide on in-memory persistence +individually in all of these places: + +- Blob LokiJS +- Blob Extent store +- Queue LokiJS +- Queue Extent store +- Table LokiJS + +The current proposal allows only two options: allow these places to use disk-persistence (when `--inMemoryPersistence` +is not specified) or none of these places to use disk-persistence (when ``--inMemoryPersistence`` is specified). diff --git a/docs/designs/meta/resources/2023-10-in-memory-persistence/high-memory.png b/docs/designs/meta/resources/2023-10-in-memory-persistence/high-memory.png new file mode 100644 index 0000000000000000000000000000000000000000..404282f27ecbe243629702b31c86c5f91fee6f2e GIT binary patch literal 106165 zcmcG$1yq#l+CPj4h#;YKmvkyQ;DC|>(%m5v(hQw~beD7pgMc7ij#7ej3=JZJ-*M%<#HHip1GduzVBaMgefb^;9!wsp`f7P$Uc@-MM1#`M?pb- zkBJIA((5C1iE=H=TUPRsy8GmM!*SCyDHnl@i)$D7O=YG}yhn+Rm=iy_iep@}CGRi} z2)mB)wNPk4j=`Ayd8{j~)on705KTfUNt=fQ*qq_60RaKVBeXe=^x+Jvf-{zPpvR(z z4}7%7OAKmUH-t|!K5nfbjx*pJ-@FP9K8{UlScAPviY{wTH}R%@4!_at(IO5%DEXd8 zqkEqxFdM@`#%HBzr_;V{q1ko#H{a8ZY!M1C#k0?Sf_`&HN7V#e zuw!%msEv6A0-jwnzYIK$UC4P1fv^{vgp48VXujR!FzE;??A;LA6!A;5 z1-CyZ-m|GxX24LfcEDX2hCYoXpl}t?y<=1T$#_t+<358fBjOeb^pSF&iK>;j2Cv< z3;*WYrxPJCB;weG>bm(wr89Nhdt+{XL%J-pY4=A2_S7-9U4v;GS@!vX0*%=2yr2O@ zy6S?aX(c~~W_&`_&KJ_r(6Vt1X>7O-&FnDp9#$a9#!vES=byP)PmQR9pRLzP^cuOh zTeu~qvPg=Z@9E`>@yHQrl~9<(fT=;g=*pe4%$a-f$@R{dC61kRO484S4R)#vz&ME+ z=gl2HC&LhnOqc$7>N*CuMMy1-D$!4**qChr)==S#A%7W5BoA~v^e{qJOsQtAdxLh!LWC- zzZN4j7y6~@f%!fR?NONo{{W^ShH^TlXH(#_903Xe8_ZL#nyi8A;PEB7hJ&X@$K4%c zL0;27qhdz;c6Lv9SKT0&D_TBlMQ+@akJr5qHD*4{w#*aLq|JDYu0VH{nx4-tpKVQG z+moIPat!!(_^bz0h#pMd5Orvao>_qa?XynlqR+?uoHH}4 zo}LP2fZa}VGq*lGH-Wd{d5Kn)BMcxjDKnzyyfZ0Yi);budwF`;pAq>7zj0+~_U)uWq63?G%cqP6-bpP$^g>}KXdDn8MBk*cby+WX?QN_2xVUo%tm-(=1w_C4N$kunBpG4oS=hf74KfB$RiUgU25Nh{HB5aYE^KF+p zeYZ~)nzVH#3sOzPOV|@CQG-D*46Mbb90&Wg=06W=OuXN0Evv{BsZ}Xn6FplED)`GGM|G7k_(^D&2T(DLh-tu*47SwA9MNiHweSCl%obdJCqR&eiCHH2RX)j?z{ zuGh)xsFK*;Kmi>sZr)e5tjd!vB}1Q1x?)_>Bi0*Kdx^~8m987q?%UUHP*z`FC{5Zo zUC%!HeEr~?8*;A?3!%AK_g#Up9=XdegG&uQCJ{4F(%sUXyn(+1@*D~kAqM&?U0LO9z9OEZXsNi0v0mX`SYBzA4H?u8iDtdjKmEz}G5z|+X`%Zt;{fvZ1Iy>OOBj!!aokuvHr?qW8poGwY#3OVdrAvkw#Y zz#O#U?&HV_{~lfLA$vdiEsCPQ5CNBe8zE`I3Qg`-TqMllssdjIr!)HNx~-o(uoFHJ%prwl9VPzVd5jA~<-grdKT>Oqa*+ug?ce9C7@#oom%31wM9{@P6{ z;y1<{mKfXS%=D-4?#G8j217?8>FzfrD+OY=TD^LoAak0=)}MoLDw*wJXI>QcSd_lu zn&qA&l=zyZQSDurnnZ?`j?;Uznl@K!oXjM9A~`}K9hSI!yTm<}pj%FoaK^308ybrz z#l9iwj>|A$V~>^wX^p;B=r6R4RtVy)0sUSvv_Z&KxnQCzFo*lby1pezd>m4cV$$2O zkqopVt@iaQ-%+SunRm%vk83aVL0i%YLSG8_#u30!IoXG~DF?4uQO=Id+RbLq7)2)` z_PqtyX$=yW_57ZrcRwF>0gLb2U;-Wt51~v2Mv!RJEjm5JcJZa1jxh2tO6<~#`9ziD zI@_ynDP3$o<5_k`GzeYVsh27@vXd(g(-_Y<{BzV}SM?m7P?ZN}wxQOo1C#nd9k_4H6>L{$TvfrK?lvXNi^%mtK+tcNM^|uaWDp|3 z!{$V;F_G$~O3_jHeGmH`&t{ut;~jFr0-OF{OE4FK_6Q%S37EM~I7u^F5*x=znABXm zHQ%hFniirHxVWc(zah80+06R2>0X?%b$`@=cWLP! z*N^rn8JdLq4*77wh-zLYaWBjlxn>k(cDm(5~`+r|(pA8_XP< zMF89oNw^CXky|4PDvnDvGMSUTUk7yE;*K)?8YdqX)XGhmv5pBm+JdUrAmg>L&F&Ve zNs~|F*`H07Z-pZ`FIkkJS3>R_nES3y5$ijNb{(H26*!r_T)BJDkLnr3c{*8Z({P^V zdGLlhbKTP8+;Tj!;kvDeEbIQ3ED^#?bMQlUFbUB_FWyGCoB9Cc_aZ$xHBUwY9Owr>#t}->9`XigbA(f9#;PgC&E_dh2sxr1?q>2gIA`65QY>Nx^-$BoIU8dub zA!UxZjI;HujVzEd>I5tmLIDTSi=+jjv4$nwffUVa2R1pz7InLf|~Yo z7jxI+vLFkKk70_2utER9he_t7=yNq(IG-q$YV-J$)qb{uqM+P|#DxaZ;gH%%{MY-2 zxHivB1PJYgFC1F25+SNK{!t9&izUMpp(?YM>cz-KD~ZT?hE*tl zoeOAb;E3GQxA4SWA{472SHM5^7ofVS7-7SmVpsi&fj!pxRb?|zcIQEBKh%Wk4p~lI z_Z!95hmn{iS1+;RgTH9DWO!j0SuoBKtJp-MI&NxqgWP4%s zTCH^qH9bKU6oRG4z`DpndiSkO&VjJ-Ru8)yP@_fn3jM|NBsUh`46EUU`JEQCuJG~t zQFejBkXLTFLEwiIgrZ;);*@p44_I%ak=2n0s?#+{m(j=uiptzw4H^yA4nuBWTLvtk zZ!9aS@rQElt4UKmTbHfD;bQ}Z6m}otIr43eC|o?R-(#bU+9@Mfv!w@D6RG^jlU01+ zR;F)F_|GLN9le(BpOd+#Uob%Q2tSZ9u!K`DMMmsdZ|^c~V&?~!plffV@GG2P8caY5 ze{BrORe7mjF?mj}PfN)<71Cn1;5UC3l;AG@6rM$}YL=NXsPvrS?x&GS79%}iRo(=S z4gmQWc=q4tj)e^NzP=4C*3VU#C8U1)s&Au^Km56!KK*&hTv6hy#{vz~ve*}v=sh=M zNN7D3ftz^s_3LT>c;L4S`~4Ze9th1UZUqQq1^vXj4t3Rue!%PT9CFJ*itB|6Xbna! zmC6(xckdh+JO}^w0(Jp=`pbd`HLvvo&AW;Pz=l<31MWyaa-x(#bk z*Xer>UJ1%K?W8#^hFl8wqJTp*3IrojG*Zb}u|1oDjn$VGQe!6{rmffTOhsf7C2Ya(Td%U4sGF3GBASdE=ODm$8n@4C+M1~Q;4`O3 z=)fJx8HB{`>xjR8r(H1@dOP1n)_%I4EZd>L7^|6J`n}_P5~rRr&I6kxg%mTa?1n(? zoTdu%K5`rA6C1(wx?_F5=V34yY)5)YYfTo|M6teGlY|FcZwEysp3?;#b2W^Z=L{9=o1vq()5+G97_;6mL?#F6*%b167l19J4rh1(SLPGErXmCCfJz5PzLc@5;H(j(Yz9a6=6CVj( z_zGN}GWwDj9m5!ZoJ{it)Eiyl1A`Ti9q&?vm1bO9QWcY)L_)b38x96L=L6khldjnZ zOt)bKqdegOH?cp@+-Xhb56qmbR?sO9Uj`)xBc%MjS-b+r5UUGxl2r|SRe@{)A}vdd zT|)V&Lg;f(-lGw#8E54Ch59PqUupp@5NdaX2~DOCA8UFj;3zI)OOZ8r4r(x7nCx2{ zVmhtK*2Lg0js$yY->4s)Do=a(PAK&Uhvcl9)&OLYlcp_|SPoXc1RMNBNL}(dTCgZ@ z_;wK8gZB^2-aGL->o#hzm@#%6xfVbiJ`i4xV%I_HQ=koWLZ4rMTb(YWzWH{WSOtpn zxzK^HyQxUmHcc9Rk*JYePntzNee7{79THQROzur3!MNjJemwrz^uD8OrhO*++%zjJ z{qi)z$k25+X2x?rbE|OJYHYroEcCNy*=pb}iZf?8r?v0YHAo#PHHx=K6cM0X^Z$Pj&%Zn*%#9Juj-J7 z?o%*2JAAV?ROL`?UsOL0a0&yeNx5Z&zTP<$JHHi!O7M*Mus5P@h)~j;qu+j z*ljszQWXd@g$(k!Cq414jD?)Fde+O;-o?t@tr5B+A!DLNv5%scK{8ShI$c-+JwP9$EsRdW)2$(Wz zSVn}u^1dFY)qBPz&3p8;WHJw3gJ%+;&DVt4`r;EX=y8lrF?)-#uzS%z>Y&M()yUjc z1c5v9C49s^`~6BCG)eTeQ%qB!5SF$UKL-$|oh#2#Tuh^Ip_H_1~ipaozF`M@DoMS*b;lK2?7J$5)^ zTXp-lkl#3(jXl!qUYh+APdnw`$~pT?E5Xz23v;8_$Y_)3_L<-Lnb_st0nNo3v)E;u zlJ7wTj@IS(=0o2b6lXr$xjkM5h=XabEw}o4s#{Qz#Y8oCLW(H~&HiOjB(d?FgRCSx z)1l=i%iUX?YFFjcsP>Ao^xqcXJpCeYY`Hs*x;x~ir%s5gWXPp zkG{%+enmO`n307|QSyM8!2(A_{0Dg3mxxyDwFmj0J?Hm9F?AXEF?uPb|JOLw#9_Sqhy!|0-+qY-hs#&!AW zV(aoEEyi!lhX!VLXApAB`}zj@D$eOAhq#y5-9z{E9QDQXz81B=3rDrmh~&vtkJia_+Sfjh`NI4D|%PJ*T3Qp3Z6_w);S z3BZr*UP+SiVXw=Tt|JdFVeBf9Gj7bUl`^tMVt zL$G<0X{FbS<9i(^&(cYN2aE(R!}3&2*q+OlWipL%=q$Q+jY{uj@Ys6dUUMwaKYv+f zciENDe>pDby)QQ7yz57G_-00QgFp;CKPj?xvH~}B7(Be(3IBMx+Wpb!0OselS>Wd} zeM|yeL#up{?Y5Bs9u?+&$zq*y3W?eHsfo6xx2HvKY&k1IBPlt257jq=X^ehRbhc}(&>=ZnUg3xO{ zV-7lJwKnlfM4xc)Wr&G9878-D4+AKaF8?y7l#YESVKaBRomD8nd|7#REYgJiVx-Lz zpf=I{r#s!>tj{F+Xd_k^#wcY859pqr9QX`E`gA7FC5ZO))s1<@s@Z2$9>EuXV&#kF zSSaFbK~sJ*N?BXNBL37caLyT*@fX$71rf@4NtAS<1JGEHb70RB4=TH&=;_z> zAXy9X?Pi@JNVN6P$8zhwzCuIt0HW4KSwv0(>Jj=-#M2%b9Z)}A;?qY2^-qNqd^cm& zXd7fQV%1*!uN-f2q`d%P14CJp>XG`w07i87#5rZwJ3+5Q{WmtQtoKNrvSg!fk~wa|y~$FQ+|s{W2~}W-@oVoBZGhmmvF78pTV*rEdr4 zC2ZFE(s5Pne4MQA;xPZ~rNe=3liyn3rAND0|5j#JYqjI+X=^M(f4CfGoqz_#H$HdouG74-aOaw0HYU5Q3R81C%@aj`yar zIcW0`fcH}S)lL+A0}`GMiDC*pdzc=LRv=YN@}Sx_dY_!LpUWWrCcNsl}^y-31mKu1(IJBD4M z>zYKi8~+{G^x(Fzq)0@9?W7l35)Y?N8~48|`5wAyO&y;qU9QoTp3KrL`5uL^Z=IdL znYrBJoH_sY4EUG*a{n-!Y~y&l?s7NNqN&WyGkWm(>$;T2?jwGizzC4GDQiJ^1gS6- z;gF+n*J3eYWPZv47d-w>+MUxiSgrpH{hbL~C1xZu9E8X}hSvxi^YTmh)gA;3cRCJR~pMubDLh1tFAkc_DO7(SI^smrqlXeC9$oS1@zFWd> zyH<$rhrT?HN45{Jn+`9|Q@1|NG>OdD3)&rhG#EQZBj|W6jAW&IQzI;(d;=aa^uOiA zeA7ryoc{=Z6Bi%c@m&aQn2sxky&1`WauNAUD%$;0yf3AZEEi5+ZF)8;|3)ByHZ)7m z)ZxC!-BS9t(fm}T)RhE?UcYQf^x1^L;5*8>#1zw%*E#h(lw`Kv!kH*nq6 z6~5=5?=S_tk~-drCetpv6k?Fiwxc{Jc(j#V&M|_49Tw=5tgD27^^T!g>9k;ylPDj~ zW#Sy2og$6matPq+@|wHuo$9k|m97u&cb8NGM%TyAN}t8Pp|bjX{o~mtT!7q}jIR89 z7x8z28Fx?L%qIz-geulodOHyEY3ceTnkjWFQw`4=wIReqt!jo&)gUaXsYPWUWLkH}( zk8)qgCRedCGOoO~pK?dbyFrNB_K&0iFa3|MoScf)5~S5Fc%9YCB7>0fc$0fiDKxoj z<j#J)pS zJs;%e3T!GR;8Oes%d=>7#zUvz+7GDA1pt}-z3$V?|DdY87fz07#6B<1Rjhclt!4+` z-;&2g+zac>EDWBln=5>H##y1JBQfz-;cH?|QD~NgmAfHfXGo=trBA6MwK(5bd0zQK z`u7YrhVhY4*#tPokg_5;SCXk@XOW3y@A1L%74pPpJM&a zSeFJgwZ2OtG1jm310H7v-{>K7P4@|l(*KnANID|PC~Sp1l@7Zqenyv!6(4YkMoE5i ziP|lHbBW9?LeQC-c$q(W)5}-i{Ko7_XmDN6-6;O0CcFhvxivIl7d2~Y6HBmK= z;<;}gA~I0-&Pn|7;G7y{2N}^PD#BX5{kk5+-!(-j56KX@exv)OlG_05pIKVsrj%X_ zP8UYfa3q2h*;RZ#AIaLQ@+`=aTY6Vx7~+DisQ$ZHANZvJL2uW;42w-)y9XPssCK%m zYk4%3dason_g2K4nzSAw&H_aI3K4K9>s=>;#-Qmz>;jh{E^iRa0?qnd1&9`D_L#O# zoz`e&Z@hXVB!kYvIM>5=-@_Xng!lsM>Ftb(R{DgMA+Z3x9;xS z@2OmI8GY1?!neN3hfNoOoT6h#5ojU=t@ggw;Hz8fBYPJ^kGS^E=c{Lvy1 z^yJc&Ppl$OJtktxx(lI5$JOA%W2~GOhw`Mk7N|K{52x+{#p}C)LKSKg|Yia#b*W(+DoA?4H;8KUrmqJ_@Mc9^M8THa|<+PWPJ@s7n zqe4#N$WI$d5{p38scQ(o&kN+aY-x{c1K%cFb5Git&0ipjCRQa4Rf!ue6z|C?o~({$ zb!S1ENS)t$$o9`r3*P?P_+wozpkc*mk4OnOlRss1wP&LNp#kyx(cY8}Tj*13hKE2> z;ylA7h9={OAnb!UUj)QVkT*F{?OFP6YUCjbvzv;{`_VTpU8~@dB@GgH4{ZIs)_EF`XoUrzxT7PNxwX%ow1cG zb+c&HX}L(CKc&Tmvn9w>UKnCwY0c@7h7>Q5=;&tiMuC?yEylu?;*6_&o~{X9z5}N*LA#Y3W`EHdJX&Hs@%QrN@NlF_5Gw z0(wpnM-SUZQEm}{4K^At{?39@%lWefr6k>Q60<%GnQiDE%?J2LTrxM1C}5NcaX2Yj z;B)P_rq*OIWxP^@-EsmmB}QJ9AXxzt8-1O;48b8P$x#uUa%R+d@?~3a^(SXrG~&;n zl=QS@&z+M^$Z+2=>>n7@{m%E>**f2}@W}EI!a=0~I_03VneA_R zaKif3T>1teDwMD^Z3`%8%4aID4o$BMd^kgy(Trz=AWiKqc1u`mxoD9j98!_Tb_`6v=6Bw-vj zU7kw;Ue_|{A1)TKBL*_2yDL5k0WdtlKOj5OkVO7(88%)1--7R-FOZ91dJLrao*H3) zkg`mwe?H2#K2>Y`H8PdzTy(tq6}y*ngH!Rk275DCrHzqC)gF5ptCsIkS?jF)D5mi6)r4guxYa?c8bp?aZ^ zgHj(quDcxh5^#-(F4ovSMbyN`8|3x-_y(o|D(VCX<>U0IHpxkWb=uXKqr$!jSszU2({G&3~t@!O2f(2Rp6=SB@fwU=tg;wl1q zto22KnuWf7*pKYMZx04@ri%N{-(8=!1h6@+=16u<+W+P!-Bd34>UP5NKln+cGPZ0& z=V3*^GAcm70!!6r2vV+ZQun3cTPM*Ia)8qvzS zVlishmJ=jJEws=bu&Q&H6Xu^l)H5VgLVKNF z*%Lk68Li^MUlWvD{{CUSK2JL%vL`*ya3^-qAu&yM#@j~pHrK5JjX6c?4eKJI5WvR> z<}S+se2iBVR1+R^L37Fhxf75|CPf^_{AXRW@jMQrZ?6JytTA%CF3`LfHnUg;&;G$_ z{ZP%)r(16C$wIOy0)Q+;B19u|av?H&X+r&;nqHnjbU0VoZ|Ll5>j4-Ii6X`m_NvkN zU_DbiBC!podCX3qZ7-)i^b}%?s)aoN$F1X{sJlfisE|)~#Gh{KY%DnfV06N{Q0Xyn zoD=o024G|Jt~-r6=dBsNS+aKalu8--21R{5uwa441P zy-zAj_?)D_H^{vDANCD%1u_~NS7??cla|w zYj(rkZMuJug}PGUbB&3mZ7C3NO2&MnmZ733*!?katW@N#5aovL#+j|%MT<%GAIyew z-YuX6DxX$Jv`wZW2EFGC6%?hC<$lBFz(NbnS_Az9l&kXn8&F<>{W~b%W`3_ zu2F8d2w(CrJ36&0P=N)}p!5YQqe&8(DkhL2vVyU<#WqF%{0S*o{_oC)?l(DoPnj>2 znc23U{nLcGcRQXt?djh`X(p+3qa1fqDuJ)Qh6!7!OvGknD9;h zU*)+xH~1>d!zL3Wx&Hykfg>^(@d4luy15ynoxNo|KklY}1zOO_gC%aCm^?@&PqkoS zEU!RU+qn49GKSK35qEI4bx$0AT-C}~GRyQMwC?zR(l125;0!I7x0;TK(5jBVAAJyP>=g{x z?x;${6J#uNsE%!Uh-6Mgy0`|TOjhcWmCy^pjsSo)7A{|FACP^s)zODVq~)YA%6^Hwr*W(YFo_Lw`G zlZFuj3Q>+mSJ;l8I~ZOTzrBbGXfpXnqIaWU7z6<1!1{~Q1~I0>a{=N`3rsr z;QtQ4Yn7o+A&+Uk2NvGgUIC>?eDfT1QaN}G2kkiJ=`)v2EjL@z%JW8|{?G-|WtG%= zLuoz8t*il?v4DcnKyK&%=m9{1tqq900jDGoJi02GXDRQM?ymh;-93(|^MBLb<^29u zch4Ccd+is$AOSUw&8rNWT>HiCK#B8hPOlF2rgM57@)d%jd$1E2*$sC!%O)ggG^as? z2Ph_LM@Yt7IkU``2$)`f#o5Iqf56#5N9D=OPP3E6eJjc>Irp1T9waAsX;sjQZ%f4S zgDTq7A~OIN8c&N9@hr{VywI0Gd@`Q@-5UY% z&hACc)GKXy8K&yFDI<0=NFj4HR@aE0yAdzX#ae%`42@|&evdUSdQcHVG)fOF-r$E_ z_cj8KS+}|5;w1pd&bXZ*gkB+Au92j1)yLa%# zg66h#OXX_>irq`o>)*^TZPB$sZGA{6l;c5|^sgdh32zWt=j#@Jxuo3gpRgrKZ}!Ax zN$^oJ%;Rf>T++LZ-Zzw;zDGkWa>2DL)&C5O2k7gE9_(Ftvi2!{T%iHLNgbF5$zEl-?yRh zJ(@MH**3CCxXmywy@P#PL4bI{|MWh`^<`z;(v#*JqQ)MTQ}8Emd8M@`A#{G zA>L_MS&mt(9Sa>t_yT!Eib^ST??9aFR`sL<|_C^KJ#)m;|SK=cU0unOG1PY zRS`JQNnCCVzDsF4@z24Q$N0f4`uGnoh(w$iX;Wzn>l~*2@ql@#FPRYpP(XT5u6GZ$ zX^(`;y;K^Xt0x+K&A1;UB{jh3@he)1G#vsL72pj=ZQPAjq-DTp9P$<Eg<_RM&0}zeb>$kmMk2_A%~1oA^Frj|=WlSF_lJDembD6=#-6&JI8x zWD}XAyN|5CBuL3-2l~(R2gRxG0H=7>f=02X33K5A8L!BRPw6Xmt%GIHMa?h8nlB8@ zzBK-M-l=?bFZlLrHeQBjkG96~Be5D-2~;T_Co~WTdjZrAsQPk3*2k^6AZ6?okmQHB zSJ(v=#ZUzPsrlN6`-$)Z#Adqg3cs%G*d_~b1!-dzH~lM3+rk;7Qz{u7Ja)~!TQUP! zQIy>0+oSYqZ65J&m(L2C#It%I+%XNhXiwv}E|0oj@y%v>2{Ks7xU{s?O_6F;!mI$- zV1H3&z=C!DfF_l5%ri8;rQciuDW$vf;nP0|Y;svO_;Y-(+c5Vr^u(u+Sw{QG@+Li1 zdtPiBU6kUXMo#WU1LbS{BEA<<@g~)t?^kZiKU06Qp&)x8FzE1-pnH_b%-OH5mVMK<~xh2K0Ev{~FM{Y`b}OVKlu1Y!MibY3wPb zWGjNNS)Mm80YOnJ&uAUrz#kln$*Kjp{SB1gz4e1~n`^({i4^(Md<<+F1DKBqY*pxL z5Q^Mk3r)GMxmH+r_cu8PX>jf7+wW10zLe0Ib%x`L-xdryg z$2&d&Xu#)GhnpPv%|Z&7_~#jBy+J_ku>8vozLBh%q@ou@c!sU)?WeciIV?~u?hoDi z9Zu+6etqW%tU8IUiaCCSz}gFJmjT{| zDtmR=0tPGzcL$G7hsEbtiLg$&M1t3fbz9D?^M#hoPDv6^N}E>Q-dj=aziVE8bY+z< z8flaW^i2ovU>QYmGB`hf?yFdi;V9j}__Nh4H2F*S0 z7WC+3#a9h3f6Y5^ZLB=7d+*l}%*}#d&IX3*xwI?=kR#8`(4H4%Glc9`97`5H6e_vn zwqt3e)&@d~_}(6izYvCI+s`1KtPVjzu+VVj!V$UY#~Oep)L}7(KVVD5Tx}l`KPaz> zSuFy(>I}a`zhipKffpBvpj!r65Jkc@UHbxk6>o>1*s|Pkd1%cirsWLiEk(cCWLJFr zU-}v_y#LYHxLeg#@$d3*r8N($YZcbiI@yWX^a^*c|1do3K1EtRk$16buq~1#X*KJY z7L7&H54C@!rPDhy!~;RNZAPl>zXajp<^FXL?m^TRcdAUhq;Xx&nYko~s8B(Hp@QSnGX9bcY?tfqtFIe>SQfh?G zP_(D0*Dr~JJ37mjrc?=X5gAEJ-NhQ+L_0eoA(Uc^mWc_q>punFN+&eCkmqVcGA-m|Bg5jeSKS8{YEO+FgGo0&EQY$@W19x9)5pGg(>>?b-fU>H|B3$Tt2fIh8*PM zGrsizuGhg$u~}@%dUF`!tn8RyQuyzpVPAplXK}dZ0A(LX+kyrdS5RINu0cTrib}%= zC$HN+k>Xsb@iy%%JX1<@h_Q%f}M_DiAWaX4P%25 zaNObQ;B-g&(XJ9r9AR;lF+FDVZR~e?xSUw;xO)JwxYB{_o?*k9D3e;$c3M@{oj z0o;nnC!au1x|{v1Vul11kZwgE_%Dem-@Z)%{Nw>5du1p}e|#21KVdq^z}ilFe+d5_ zDVLHk9x|*JYe=alDmUSXn?s46_MNKLY6k+CvkJCO!r2)l4+?A=`$cw~Hu?K=?q&$K zkv!WReX2?q-h$EJF%w_k@?m`Ha=STK01c4Qd_4Q}x9be6^%KEBI;)=YXC9b`ymROl z^kMzj8@iPFuB7TEC*;1rB~DipD4zCw-kB9R5Q$ti7n?PS%_V;1&*KTcdkA7~Bc2dK z)teS>xSD*dL2}IY72EfvQty>*WG!$@%AfB}#$R(h`P11rrD#_goAnaZ28T%#Ncz53 zkO>&3|B9VYVY}Gr@m==~2Tw0^<$3w8rFMvT?R9qyu81|!OIl*0(rBS5YiS*XcUq@mxP*oV^gBaBCSG);t#RH;{7ggrrA~10V z2#)pIr8~YoB8g=E)o%wJJxnUi+N?L5JYlHphLG$msDyP@o*?%6zhp~)b|sz@@Ecp- zNUzt*2^ZK}JiIiKkg~QM!4Wi&B~}wSi~3caW38yS0&&!n;f3x)cbT*5wF)QnDPd+0 zGaCP_k)>u?yEK~{TQY;kS)HZu(M)cdMDP+X4JwiG@(N=zR@!9-s6A(pe5Y^s>Lyao zN6L|aU%+J{%iD9^*cYolB7Q*`z{PYg!t8?$HD3~enMb;WLD1|G>Bj+9^(u|ENK@d7 zPk7pmzd>;9qJl{JZjmD+JP#rU!hdLSznc8%E&!_;;64?|9Tyqz`IYdwI*)#4w67k$ zQnHZO@F&9piSPgFR{VO?N$nktVXqJ6wTYj<43cbV zlrKSkFiAM_eFh-}c?j{QiZb+X^nCJ$HP4A$?SSx4dvCOFw)JB7EAR(Hn8ra7kuix* zZCSkzpy?;Fi=B;|U@U{)2{)La6?XSN))_eEM}vyncL$DO44IV$%>r6El6Nevz+|uj zc-+_HJ~5y^@VoVbB zSrV>*8_O^)6dtd-^szTN*>tnu)aV(D8i@#eJu99%pCJp?=DczK*)qy|FpU+`( zoS+~LjPk!281PON{~)9I|68_5yyXoUI9Vi-J@!6Ujb#reW8CdO2aL!$|AfOC?7n5i zg}m$4$7=>`jA*CU(v~ik*gv`$*Z+&Dn4d02Ar{fTOfKbw&usjM!(th6wtA@%%U?ry zMVjgPEP&aAfA-uGXNd5bB27N@7o>a*yW%L>dz2hn6SfD80#4`&7-XEfzKr?SX2Be{ zjC`#Z1*fluT6p+8`1Z0nH{Mv$&Jov>C_JyKYDG<;QWtdD9y`hY*8B%s2bHLPRe_Wfvf>8i4D=9xe*%=aXpz5r#9c&Mf5U z`}ZYa7xe%5R}lOvT;%3t+U286e?U7T@if2*{Ni}Rn_iuE1ZQ($urWUzNX6<~AMJF@ z{+#Dj2q6iVxEXE=?@ANm|D2k{7$V;Stc-sJ#q)A69xsGgE(#y~We7cL=8gI-!GrIG z*L4|e9ySUUg6ul7O7;!1{)j&(G?rmrH&Q77>}D{a#yg@_QyqgrSd_1!LmVJ0+M(^5 znCAMT@@D<=bSVc(&5s`Xz-*QP)hYo}APuVe@V%vlG(3Y}pCvKx2K5KVr)&>P9}?h$ zk7G|AaIXS3y&=!6=br>?Rez8T>Ly-2dst7>w9^QT>Qgg>iEp%|mfWUvDf0kL^ve#> zc}o}k!^)@{x#B7RL)a)6Iht;uF+vt5C+wG=-sC`9LSY98*T8^2D2i>pSLAIJLwEQr zQ#je@MU`jD9_e1Cffm_+tP4JP{13RC_vwGeiWUz~)+@VDB^ykRWRVtQxY(z?S1k&}k5#0eI&u9a)v z$c)jsq85U=&NW`EIFWr}(G3^m+das_Z2RS>?E7pHb4qbFco+cXDi(Ly{$;#Ks?CAm zEzbPuyB3bvYguO}qtD+w`8`o}b=LjYxcr}{HA>KRg_Zc&b>KnjGK6_W`E}yatR+sRF@&{Z`#S6=US;jA~_?^!*9ph1Ly=m5<03?Q>HCAl0lcJbi~=T>9C=2SF1H>C#3VlL1ONRM}b ze(w)6d0hzT=R{Aet~g!a;HMSU3@;FyuFxH+@>fA$lsU}KW`l`X5P1`*6f(v=dIf^` zfbLMJiHv9oAGrp%%UM-h%men#>aR45^e>267 z@c-K>c8|{rrGJhPEw%rL2vIfT>b8(8M{nRQ=UfOUc37qkEpHMwt`f6hSDp_Pd((Rv}M0W z2(RoykcFudDSpR&D7XvT1N$}vS(!f=4?I=CA50MxsPD0m2`v@&K(i5&|8&RXA?1r( zn(P8o2_NyNC_^8_IwxIc|3$JE{=3-J|Ksf~prYK@Hf}@^0YL--B}GzFl#~IaLDB-G z5ftedVrXe4mCivzL1_erp&QAeB!`ZnbLje>LHFLCbKHBK@BQBQTOez>7BTb0|M_3{ zb=|*vuo!69LDS6N&41Z}9rD^7J4SjNCVJv`arXTCj29dj8I=D7t<%tPUDOGIi^lNL z{@0ULhF{P+v7-0p?&ScXWALglVf^_QO_7?fzHFJhzlW zBMU#Fh(*y|a@-sr>Qe21V+@oB{!ak_5oJ;y{4TrylEI3slxAsVWAAuPmMEnt^H=x( zCDZJAK|A(g7alw|Uq;G*cMzz^2eD37ifi=~)x^3`O796cUf7x4tvRO>`@)47`7CUy zsJ|P`)cww|mB9CLpO@TMTurXJ&}UhLKGmEweb5!dH#%U|O-TR*@_8c?S(d#ZOrm#L zU;Vyc5WUrCzs%Zvze8GOfMuCc_o|2hErnkGbavxMK1YJOT&2I4Nz4D_O=Or${Nhb0 zj546~0BQi`jpr4%B$;o+*Y4=+RgL0wHZ0B2lZvvc-E7}v3^w_^1RR6K@wRiF%}?0^ zb?!FNMvl7}{@H4g_|OYzwaCPxA$+hp8++41z4c5L-F|we*ooqxjChoVW&xJN59q8; zx>O!mb-x8}h;A^0SDj^(8*c}D;hvwhzR4$5 zPXS65qV9h7I9rO(lHQi2uLN`C8_|o~@1`7yeXP}+huvah@OfHUja=Q?!-&6>MygHiC_&#c3B*z>5KCR zo33_Oh3?CJd}h~kKYh}i-pl!^p%>ghFlgdUw&PZ5Sr6FaULPx_K1yKXQ0vDWryTn4 zf~hhHBbc^T5K%~ug_^6s^DpKk?`pXp_PdgMuOu?j)|uE7$CcOs9>#Y1Z#)b}(G|x~ z*^PI_47o&7=3vbBN}4UHu6OSw+?h$Cp*tmaeElh(Q(oQQwPai)Pc-r(Z>qf3p|QC* zd$u5@{#69k8Uh{<$k5XuC^fgJ2Al5$HjpP|{#T}#mtjGXI{4YOzhQbM`b|dmf(_H6 znxD^P8_Ka&QFrBGfM%;cCpoyFg8(43{b2Q}WSVq0SRsl2&F+*&l}=>$(<}G521Z;r zX^+Q@A6Q&1?4#-cKoQsWzb9&Altb^Pf^+c$xTKy3>Yb}CO^97jbP}egsa#Dxv}VP| zK66RXjl5&@?}%QkNe>H{^Pon7Pc(1+lBODnf>JkXaghiErmGSnNrF?o&+y~Ba8vdy4V)$K!se_je&0NDc>y8AA+7S634 z@hRfYe#d_4N-oM5VSrxdk!y9> zYz-j}zEH?l&qk8xV0v*5QplVGPyBK5%X6i?o10e~uohxpM7DY5aJKtTDL2zwX?oeI z1ajru;*?bUL|>T|)0BM)F?wR78)TC*n41JQ3XIgo#5jHgI7mb(d-$H&HKm{)Q3fVa zV~fHT@@YB5gib-4tG{O_l~ixs*e|qH1e!^z~*^JE7l%c37*x@xk@x>!TSTA$4mOh ziyBUzD@7s6dqb+82fg&;>rQwk^Ki;U*Ipt)q|Ipg=@;++uw^9H%0OWrt+8?45^V1s zmg=qop|a<*c3Dbiau9H2{Nm&}J$|w>?ooLkL2LJ|DbE{vxIvUb zmza(U1?_%vcCqqGA`^xAgc}XxZatpRjbsV{I=U~NQ*yhDE~Lty7fw;>Z@GS)E+9#& zG_9<1>#X}4XhFFx%GypLYT|1ic%JY&{E{f_ex~0!o3yS{mdoXkbA$DcvTZAa9gVE3 z@^pcAX?ZNXu;azsMONDdsLo#d>ci2sli9@Y_%5pn<2b^*Ax`vLkx4B%TSv$+&!cI2 zU34LisMCzIpciyxvyAn~x*HtCZvFQb6Tl7Pb>VJ9aE&FWqbRNcoE}}bhIzPb=2+*e zvOTrM3nJg(D<~H16ZK@*!?n2$9+Nz)F7WBe+BjtQ3uFqO##6PFTMkfZgY>9+!H3EA zjUD+=apx&ANlfFT_(7WZmroek_u~_zy)dmZsVHoA&ZG${rjogNVJES&`2$jJ9^1@k zFxqA0_qc^k_2exc;2l{qdRXy_IyLgXbzlfR*#n90rV;9w#X?6;l{Ju;TviiNT%1He zFZ93qV*n@Rx(m0&NB%e%4*5W#mh>mzK0c=${1Oo!Gegv>>-a!>M=$tvJ8pm}#{U)f z$U$Kkq8IGMy$&W^Mj|@^^tJOz<(XObL*<_B-+?VCi!y?LuYhGMdSdDk2Pi$pZ9WOA z__D-*5Q=kju`8g)SZhsAAVS_zH`R~ z+ui|l;uunyhnDM<&$)#Ag0Ru@zlvr1G9G=IoMdZ97u^7AtbZVOck^xA{7}l8(P)~w z_YHd&+X&0os>VhIa!EGcWC0_DDq7b2arqzca7v7j_p=Ct=z(+v)SZ_ zjkzo#6LzWx`u?7OJ=pqB{A;xQ!Vms6Ifj3|aQto_K8wpv9f@b=^OLz($@Y34e*F_F zc2@2hVha6`_=D+LOO#pr($F6dBoOb2ZLbi)8jSI>WIW8 zNyLg{*9gpg&2=PC^z{4y(s3zAd%Wnhl2s02o4~8D^fi#PB#}Sv(8FBcz2I*_es=Yg zj%K|AFLKyZQEF6BO5Dwq&Kbns|Xg9EnLCbABxLG~cll z8XV3`SxNs}P2S`d#bL7#aQwn)&p3WpU;oPS<8}Hk`mQMc;`sg2cO~VW_PlhG#K$qq zwc3R#y-r&?iXH<~jr4q4Jj@qD(8}S)UlM%#h3Jp&?t>Im!}i6tE2*gA1>sHWa{&}A zu;};TrRM~lM&}=+bwZQc04Wr!xzT4@jwPy56Y;U7+ve6Ak?L*aEAFAwk9wS`>v9S2IhKk{R@)2 z6V%vqjlh`x`wF`pTTXKGXDqbBP3`I|fTpPR5Zz+?34XmZMU6&bf5(IEPi_BCja*PL zn;as=eNVddcB1-Q5$B@3&r>uQ{xpz^`q^`gdMg-8) z8u321{CE)@rdv55$&E%@l^=KDrlSPt`K#TQq<^r!;{euoyTlpm`^ZD_pV5B_A4j5~ z00~Tt0RRhY2xSp+TXgVkG#n`+38{+%hmER-2k%m>( z#&im#w)Oytj-Y(?_DaE$4HV8sDQ6RgaZHBYwON2)1|CUo|Du6Yy$5LE zHUJG=9o!=FHfwH zt2pXpXeJb*noZ{t`Qb)o16;q!d4OM0{N+{Ao%?{R8hE2!Arbjk*t2+6&E&~oqP;9E zs>0=Xaq>_43BVNMR(Qw%yjPOlLQZcf_49nHcSYORa}Ej*xKbxu=Y`(`2k22t&OoAD zO=S$ZXmgy}f5Fb%SN}C7Nt6iv6Pg#k&;l&B04N+N?EGviyJ4{(|5gexcmAVzpnb@u z-=)KN(|C#9Kj9y+c@AXep`ZWtzVY5Gj@8lru)c}bS=_`u;A~6!c-iLZfCKH1cSOno zvl{nU!4?tZ7NyK#ZhaQgendc~8V(Op4q%T7AbnZQz}qEZspdU_??2owE=X#o~41w zkh2!o#W_Ok8Q{84tq~6pyy{5gvN{oyAD&UGl0QQDRs%v$LN=9{fDtr>Cz)wKE^WrG#-Y{b-;5u^%mz^{;+3zdlj|(@!@f__~&Z|%Cj zb(wOS5ukX@0kMU40@2+~)uS0!aX3aU?bXwtOyG&{t>Sw6;Lp3ctU?K2eMA|kY%FDU zf~>Z1YC);`Ey7D%Z!N@^K~((=@2^mO)>-7c)bOTj@v|&0MHyPf z9PqG!?%Y*lF!?aAPH;G-Y;r?{)mx!1_l(d3?Yxfbz4#!&w&d zjc1u?0 zq!GoB?3Bt40`dMa;Q0)u0t7sje4$b{I_AbSDmA9hLwy-~!LHV2S7(NytQAC_yTf}D zY8GLGVAZ4D$-``ml$jl51nRE6RdSnEou{R0n1Oq?%b78hiX!0nt#ZFj6r1!VYt5@%uOsw492uXgOAEdCpdxBsnEu0elD(In3I!dfPQ4$)?#(fU z+u!&`w;f2fq0hLvK3+Mt{eFLI{6sIY#`2_;_4!8d^4C;5w7xwy&-k-SShWTwrfFpc zeEn(o*4d?^Joel4PxhL{S7V@z;{hZf<8R(p9LY_@tUf7ZM$9fix)|)iv=Ho0_!e|1 zYBt)^{q)iYqdsbGm>mT) z*X44!0t>iRP$G%D`&x=`m{=~abcz7yc!TedhX)7oZF%@M$z^KLInP>oEp?tRzOB-% zTUFWYt8HJj@CSQu>1xDU*e&gA@7|j>+FNnoY5Xv_GhF0%I=YmoX!E9eKMAFS9U823 z9T6w)Iq6%8D7Vl1$_*7~l_+y}MwzPjVUw90T3%Mwv_FyD-ht2417wkZ4e?sBuE`d@ zjmwyT%E!Hrs7t8|*RBQgf+?i03DhJRvab3atH;?~AG+;Aj2dAev7wj>*nAc_8p=fV zI-a`Q|K7U9H_7nFM#8cP!vq-NZrc5Fo-pt3uXpR8`vGlqTjFXKCAzzWar`#UF*;$@ zExo~WZL6k7YM+4H04S&_B69Sr={USQptHPL2(i$<_cQ~XHq(wnc& zXuBZ9unK_Qh0VFAwQSa~ZuUEV!k+h+M{FNUebY|5#bN#a0+7M+U1#T@Iih6~m?nx)?BDF~IFvQ-CXK&g%?3v3 zqQeX~7v6b3up*a601nvyZ3$wLxc0J_Q1)z^8G5yQ`;Tn$tp{I-P9&$82_|;2dRaMH z8pZRNHO;ID;a$8FjiGK6(4*RE_PfX=kEna)7?mYbN zqB-CW6n`*3*I z5C*Au6F@3z5&el&eE3>P^TtUub=Cg}+OjL(q!SGORzfKHRPH~z4&@8Zx(*2?3hNiw z*zV`zO?dR5lLOigYg^a?CL#Yt-QXAg|DSb(wrv6OdZJ$^ZhbkrkMW|9vc7!(H1ttT zZ!)Xkg|98Ti`U@C$g;;DQTHGEW$`k>g%e3|dqaQ%U0VuO{cIOlN4l=xu6SXLJzN`( z&8ZOr?0xRn47nE=vW^Vrb!1@LBP0W%hkc)vlZRWHDs6f*RPLGfsA4tdF4gSI)9p~zd;|&B7|E9{|0Sx{MCM^tG zbc|yBEzyyN{0Gr-iJ1Sqrq=I(j+nN`Ux1DgeTqO57wlnjzp3L;6gg**H2aAoSeVvrgLi0BHH@k<;sh z5tlJ-E&MBL*2b-NnR6I4R^kmD`#}(^)9V5--tx@1eE!ttnowH==$v3Re`?#@f?Sg#}(Y*CK>nc=xD@8ke~1p=op?ABk!3u$+P3Q%wWz!H%0k zH?u}Tm;qY}#M}61T}jV$Z44)y*Z(8Yoz#N~C@9B6jf|p6(texdhIa+4k5_H$F?(8K`HutiGrjNCRiS`@2D727#hyQwQ?2E+4yco#A?ba$7%u<{A-O zFNqO9wc!=Rbce{Q8H6D{}k!VP? z0AtUOX#o_*+|C()H!VPv==akC{-E3NJ#-1}e5644G0#Hy zKO+rE$~E+r`NA&TLqBGe9VF#;s#>iu{eDmY1~r+~N0bp}_Ov?__W7EXOL9xWRh;`XLV3AKrK~AeW;kEO;;;*<1_@wir>p!96^Brji%h~b- zCRQkt^Nl*;&WWz8o$_VQ-rNNO4ZxJmqlw_^mha@L&k`BS`u8}3Cc49AeUPX{i^I63}zMSA7RLp(v%Cefq=K?W(0M*VNba8@nLAaTJL3gmY z``&Bve@{&8vypcT``U57@^-qE(UK?Bz((C?DfRl3zx0+jDzan1Sx&FBvPn47ck9z$ zYI~)N61()n!J7R&?Qjjv-FT7vMq0RoU!yXWbI<5H|D$R~Rxxbh9uk0CTCk1T13~o5 z+MNmoxB{Uy(XgKwnH)^+#5R~9*Zd0iRqdWQ5f{x@HA-iU{huNf$B>iHXTybkXEVt; z_c|xT{}O5EUerI4b{qo2B{;4u^j@t@`+Wi(J@YexZoM6-T;*pLu>^FWa93#j9bxBR zdq&Hy{OlPux9)W)HS{DEcv9pqJ8IJ#Wz74cemwKXbh${+@3CfN3n@8o0$G*qrdj~z z>TeE?+3F{~7GXkYc|la7_9K`7n5@Pn^1I1u#M7LZq&Pb>r{hek@_8rG4(fwW&No!_hu-`!UjuYH44lA}q#@`)MaMM+W7U7*;c zX&;tff%OrvhB0b*K;$n$oX>}*84Ll2Z%YM-zU8GqR0Q26MK7E-$ml1> z*;G%to{#lvJs^19Qx39z5wIx_edy31$QeJ7L252yK$zXs$Tg2$5uittuZ-}g*v@aC z7seV3E!6RwL>1MsdK#PuV*7C^0Jo>ESL3vCPWM`u_+?LO$0Mjafkzhf znBoX(x7V^&Eq*v+BDUMddODPle3UInyV)){<+Ofx?De>Ws=jAeul{a&MiOMgu$eyp zp!8THZ{)PgD0weZW9@K8W5x9w2fgQd8~tv2jMz?1wR8J9bM;lHJ~360-I}|8{=jrH z3^jeSX^PNF>(>1lBJiD+L*UnK`T?u(%Xv7N>f!dClgcfP?loVh6FZdk^k}p7QRC6t znrG;^`RQuL9<9@2*oO9QXnCmn;pGk2j*(TjZSCsQMWS($IW8xft>~4!qmdE4eb~51 zW5UV)e8kDZ$Gqx&<|XTc1hVSFxji0Q2c-DXc7eXo=Jx^6=~G|`ePfiTGx~*G1v*^Z zbE6r0oEmmCr0+R@od)9hq()x7|2bw)j{PN&)xaN5tCs|@!6&iZ`NDa3|p0FykFwb$r6`uXQT%XrV%N^=0Vj>7s?Iy($KUl&S1f|NmIm^g0}Z~KIJYl? z@xAo|%syPPy7WQd3fcp|P0}*|o&2EF=WWa#akjQEcY!D1dZjye9jYnmcZU|u5BJ>f z9nd-}?UiqS)6m`U)^ImF*=P2kjJ`0{`e~j=y=*3>}P3l8sj$POc%h*I{dJ>-wvZi3Hf_ zsVZ!2Hw=c{+Lmgra$Ns5PE&fAC+>#Qcc0(kai8etsW^>BjN8nE*F^V*>UZ5=+8#H3 zrm$Cx%y65iId=rvm}(>f9)Xp4$A`PBgWF$n3I{B)27n<9D~)$7-MNIfmQIB%u%Pu{ zdJ&yCkB4HlqS0n1QWCF1&)Pcj-6W5U!K{1eIE-YL^HJT*T z7AP7HMy=yJ;pbwx)R`clqMj$9La&7FabNt$kTos?zvGacoTP=*cPu4aQyptTDqO{t zm7{v!j?5}wfq)lM;J|Rp9t+aOSx~E+ed{O|m2ORWz>rC-M|e7w@3(V4BPXu|OYpJi zobA(o=!-#tH-m{6ZZqU-l2nr0$C5G3GK5HMktbvpZ=GcCX?eZ7IIcq?00d4)^Ahlj z*)yJ-sy^uc9Gg*mi)HZmMC;v9_9USSDKvsreCHT`MUxW`;h^#O-Ho86E?-^st9ujNj&Gky zE2tbe@&=UN%=Oc%KQ~AL9SI`%at9J^OJLz+QsAiml&TKA^cE%58g_@1wCJIifhpux z*B~Ui(#qQ$w{Am-P{FIDYx(`j#9QD+SH6hjXaMtPq?Q-slP*I%xjHR;QJ7Z?yl7vP zFN&r9tY13d88e-BM?!Yv6-{X@57kFLo(h-4#=fWyA&kOnl9&Qcv~9z0oLJhr#Yydb zhFiDlI7lNH_>nxMGs;6;m&OJ%=_K!rD2hGEWGt%1uenByYq%!TWC60A68EZio};w=?QAx5m2*i6o^SV~UWFngO^)Wvn{IG#8k~ zHE*Spu3+)K+VK%!_dp+&;L*_9ME3{TExA^);5<4Z6+Y3!%ea}JxtDGAwIwO=L-E4S zqkCieg#6TJr!ZW08Xk}==AEw@g4V9hdlG zp}BD=C(T(S!0X+W<-2z(6jqBPg~cg#dyoA8wS$Ei) zc9I~vbrNkR)9#q@-Isx^K;K?15}SKD`@^FP;$gj5gRBY%iAF>>7lfl7}HU!nNQ=^TsPri~Ze)2Rc9yE84`snpE$v2koEc zp5Wvt!(w_iPP8(IE+Q~DH6In?oab^*wG)sV^eLKOhmDuoFZ8r!G_Rgra*seLKyg17 znpa5Eem)Ac1YDiIC~va2FNjZLAjhlXeCtcM5N<`%`m#96Y}}8ZJ!qA^O8p64yjjvNiVNZ2v>@M?VN(`5v(>OK6L+0^6~IfSDklT*B)l0SW1toHQ^=H6rajE4~2iaOnG z;W@EVJz~gOfvomWkBcR)+&kXs+APSWijct5m zLN|aJe4C#?07un3oTq=lfXk^*11Z$E_r%~~ZT{?y0Ere=y<3MSQzf1vMMV#NSmKpt z^vl(arq`O8xK|x|15o|7pEI95fuXukPx`bHR&hjrx!GLSY4Qj9@A9m^6*n#>M^qkd zw&997#dYE%Fdtm>n9&Z@M27VT{X_udsdw)h_e|V*(KjPnsh2i!ny26o`D{`kotNVzSR1|KrW`s)0t-Tg1Rr=?ALk4ne*S7^)v8Heiqy076plune zC1epoe(`|Xq94#$(b`&T_2DeTgZ1BdZ!I;qJfiA{a(gNu)>f*|U7X0t^r#-(QoOm5 zV&p7`2hIAfq0{ERO3R>qq+qe{^W(KCF$Qxx@O4XG) zRXxMp6F(k1^ln~cCAvi|TPMz`m zok6g#map4aZagY4MBOnc->kk%gL+)*5~W+tZ6q=hXH&a;>IXcT)V(ny=}sv$G~{2} zsvNxxWu63D3~>8j+U`hj2>CPetXHGu77{2>Mon)8gn5al9Tipip%cPvMB+yF-Q~0* zrNwVhQ-^1OCB3Cm3Rg6oU2xDWKI#?|$RG@}&> zf2T+AzR#B%hq4rNDEA!Kdo{>23JB@I$sxWE z`s}Vn9?e8ejT0-JBNoVSG$&6J(#|D^omPDa=jS`n6EtE7x=wwfOjXHm%Z?osh=}jC zAQj2}-N9pSpOc)(O@(D?e}+H|v9je%qb(S)a#cI{ zu6#ICj#>$}(X`M3v$@e2@gqgu%iKx~U*D6mYiPfa45#oz0clq*t3;6V=64PK#?wOU zFrp(KkE=}Kz;zvl%Dwce`1BhV479L#&CF%4G53a>3Dmfdb5>d!j_U-=Qdpc=5I7` z)7*H%gpbI%iqHip4ffoVCNIPW0#O(6U@tGOZhkKJA0_y)x~9t+8gXx4;R|O{2p0SY2osgBT5*YEU{tgoE;fHjooN zB+kCbPB>5XQ4>qI@FbiC6jKPlh&T?0td2Fmgsd9XfW?J&9!|j5Xon7V+aRkcv>>O$ z^N4ZBQ$efBHF(0>{-Kmywd4BwY0@OzMBM)H`V@S36pb37KWd*m6w85?1F!AuGvO@? zYa*~-S6K*)*WML-9wZo16TOaPnGzPyFMd6Cgk%3vMh9;l`JG(LDMPIh3~~csD8anZ zpFayxKVI^$sI&gv)s$ZMjj1^d* zfsa3an2!KKKE1F>&wE*y)L8I=j{S4!RjLgg&fp$V@LX_y3`WLC+gnBa`=1nqO3mwt z%@;qsSS1xd@m$CMxgR8Sh-8NpN(OyApNRP}0*3>%Y4*huXK3@Mu^6TW+j$CtNtg;d znF1oFh_xMqKTo=~3Nd=XqwtUF`!_dwmW7PvB z9K7dSj-!DkI{6jQkK&$mLp2=-)^)zODt0OI>rR^b`zcD^|M%)qL!ZlOcmC5KvT^>= z&0DT-Du!xadsV)AxF-l7uOW`P%t;|_jY~TxE$=LE6flItGCWGcaHR@+&PuL-r zqew$-KFsn**-(7bvU8$C>?f{E`|nL+d1eH!QETg9r1RMYbTAEPMsKJOx`G@QuZpTf z`drqU#4~ouOk1a^Fov`b`VR!HL{ByHqMcR_!HoHu#L%VOPd6keL|oIC3sYV;Z5^?Eij2YZPU%hcCErTA_K11w9hn^H|uY@za)!&r7z_> zZOasnZCSekS-atW>Sqebg0aGO@+o-yW3N*`qCNl24Xs|5QX$M414 z=JS4GsW+?9^I#F8-a`DCrv25JTx z>1|`5P8EDltk$COlA)j2W#Zq9L9fs297G<2Sn!bwcJtNbtoU=hcO>vGd6=yx+|i)z z2Fnxe2H`kAb|m!K<^K}!u6-vwjCAqh>UyK%H=&GkIQUcY_V}DQHL))%f=0uix66#@ zR%sDu!(MLrY!~z4##v{#&U7nwO2)6G+btW>jmM1dR5l;&*GW-PQ!l-DuBk=lCr-$p zN#|)@rwRB@xayP=wK<#@RSfgF4tnsG$U3~`1H&c*NNAKSSBuyuoId;NqqmM&RD2gD zsK1+}(h^0v;?ODnBE=g1K#uhN0 zN_@QJL6>nZUo{*p+)Q=<-3R7IG2zf651ITn4Y>l3=GDcl*+G{eV9PylhuqjBpFJoH zDr2cx-tFy!scGzMGWFlFx(=_4P!TDj=@0HGYp7r}u2Icupocqs`81F|G z=)40+KTS_?YXa_I1MG4LIv4LI#6Nyt{hG?eEJ2{wFCN$-7J)zmLdjIbY`KWrY+W!n zc}@?8sA(iP@tEi+NWx5DHxZbx_q8sh4$NvY<=+gBej>J;sLNXmmOm$Vz)%*xj_g_2 zkxiP|n)#6Bm@ko&b(GCtoK+q9XtdI1X(8q-qv8yWhzDL5?}pIFrq5+9=R$s*k+*cVGZab`y>{W&q54$V&e-AY(P{zlAk}H{ACD zgMDg3K4OsCDm(J|Wu%kh9N%j#Z*_^~NSB=}%UU?lrOerXl=@VSb~}}F$ij{;V+6#& z;w2YSelawFUo3rV8mNf$Epp@n6_LPkVos7mcD`S(}mveGvZ?o-3Ke*2C z7>nR(VPv?%mGLoz${e^g-V4eDxU>!a;oY)@dhpW3lRnNb1-$ex1iT*C3RA=y>I8B1 zgp=u1xhE0jm%?wn@Y`Fi68o&F1JMbq%9 z^}r^appRa`1z|Y)VAoGaRlxn}NIk&~n-jttu*fCVSH;?WMG!SdAcm1$a2{3|)BnXH;H1s&vH zr4uVe3gUDAt(i-QB_J2zadK8Vi4H`4o4EMpeqQial=e79d!Cf*$u}UaoHVR4uf#ID=vmPL;GJpL@S`bG{@CZ)I5UQ23PjIx@Z%jz4v_ z`_=-Pc78KTW3bR1FPnrLyHL^B`+>k8@NBgeBPE~{njpcz3*1)``y1*J6T-;`Rss7H z{cZzAkleiJ^G1Yc#fB&belrI@(+!&3PW-)<7&m{GPU7p-+QFzn3BS*`sWNTd<_{*| zp$EV?lv-vQvaqdOkKu7C_4h*BHaVO!aj}2MOjGj=l z_9y;vH(P*Awz_!LLM!$C`?%~=dMdq9*IHRMf%i=Xa=Jwu<8uAstXsCdjlc>rvu|%{ z)hoOF)*!jw%ld~tpHZ#%ZLdP93CI%#-2h#qIt@4Gde~`Zyc$Cfc|y&9Wtwy@m zA>!9kB$9RkM&bG;1fO_^ZYBCy&NQ!9tepwM4W(t-_1_YDzgn9UP2}hs{?Iz@(hZe- zYtuKT8#+c@tf$o1v<UBM^) z8bx{(6Ly)lP2}FXPEOU7)@oMX-ilg1cw<7Hx+o!&H=vejH{!{I=Zah}$@H^crn=`* zM`u(wi*@##l$kpFM)+OrZ@|=-(onbiEdo_!9yt}A9_hZ*H}AxWF%_r-_n56CH;Ydo z#I|6OiOCQ_imT#m_y~S!L=txzYx8ibk7y%RRJ@cmPc;p|iXNH*!OR19m+HX+We$Rb zKaSD64q)>^D*&TLjBd?`y+MU~pbs_Hk%Ug7ZmXREC)=TT)x|!j)x##P-PH+rLg>j# zZN0vz&47lOTk}_XWG&d!xxT}*zz0R_Hn-|3PF}I|?{$r9yh{dsKwaa?IU)ZD?rF~j zv7RVS031yQjJc#~T8W05foj2OR31(9Q_ZLXAMGFrcNNcB5}-KLyuis2#j=Fu$VpTi zJc&-t6aS;I@wI9>+kmpC-~+y1w`#Sb~~dhvRm4)j-If^^EnGBXL(1Czh2soz2m} zNxR5t)&;(1j#)7sxQ-+XxI*?p_}smKJ&|q>)RDv2b;O}Q-Uor^@Vu%(YdmrHryDH^h5YcL z;E8U!DY_cbT3iwZ%M8e(#-5MWw<~1cZ(1F`>of>|8Tn;HX?fkl^6C zk=Gg`8_3Kqa)t18@<7p*9b2?we^Ui*dUV+ihgF+Jx+Z>_hL`+LzmERXd7%|gp8cAy z_UuC1@+tWcNpmxS_d3jU9m;&DIe43gxeok$lKXbXQH(`Lpf294S@lV{GCE!O^_8i; zt7vYm3jq%RJ?cq_0b9f{qxHm$B`$ysf7VR7sd7;pUeS+gYWMx(EDD z_9Ll}DFF?ljiB=jfq#?h0xB>HQ=B~?=76-pOGZrj6)+S*FyU5{^aNsAy?F5#z+Y?x z|EL`RUxAl?_6>NLuYbz=zf4F#N&Qi2`1!5>RsjIM6(dA|UsMOC_D88Ci-nNa3Lv%y z`wqjIM2SZw8IJlDUff*R58v?}GGMqe?)Vw~(-Z{G3-HQsK)3#Xu~A=)SGmiXXce-$ zuOfxr>U6z1UPIT1041C(9%eD|@>tNY?&_E;p{Ajyk-0hHh&25Fg{DgV1Q9Z$l9%Od4l>26vJ z+%_=a(Yku#$E#$HMTQ-OtE<~(sNx$rBHQKW5DQo#7bI`Gfh7iCIfpn_Te0+gNeuib zdJ9uUe={7?b$D&GW5B!pC6T@j<~070d-w=TDyX(3y|rIu^WzMa?rCY^y|Xcz0rvN9CF85K1Er!h2cKW0?Xkr9RgckSt~MP8 zXqGU!gw$1PhTfES*k7!nbLO^x75_1pmEd*gw182SP&@m^pxKx)S7?pc$`qglg|Y!H znDdtw6c=N~pBkrNO%V1?jz9d={!|W4^Qiv#ro>p}Xx)T+$RY#N*UFhR-+SI7ZX2X` zI!&=8yoE@0v4d{i{&aoqxp>Aka5pR~>@7~e_SgN|1Fd-Pc0u(QoAb;j3>}V-gm-wu z*4nJK*WRn9s?HoK6`Wglnmeb;C&+mv{q50Rx(Kt1C1TKk-ljsLsS%gpT*Z##Z6CQ? z1(V8muJcKY`Ql}`);(&^ID$Lkno*5%7+TxttJ+%kFz2*Q&Ghtc zMY`3DHyX%VHSoiqx3whV2sgYJctD)7 z7yAL@wp~^yNUn0!CZGza6)=l@yHjZe>Ry6W$yQp;)nbGQ4h4DQcM}1A%t`Tf?2#3v z=1_|N{BSMrObKFEytECZ!+`GYswlIurjQaQwO|VARK1|Y1MZ5Dg*B8tJu1=;X8fSt zveu$7fhEH3+$w|jJ~YoAXYN@De!ur9r#IuqFrJi!Sz}wOlm`f72Ot)`$*AQ9zgJ8) zJ-D?~;|?=cIiJb=io8MPvk84|uFQmRJo6;>p_fl)>gZs_PxYV!9GkHB(qm;d#$0?x z`PSl)9NB`zoZW9J9ZDpy_8akoSnWSdS-WAw{BzK(DA_-Eh&9*@v=iT*;WeqHZLK7z zgRWsr1;hGtVi2ut!k<%r6~hY_x@aXKUBl@ zqL2?~QZ$cRAJ;=^YD!Pg{mhn+mvL|5b27P(s#2{HKkf*j4@Q5BhvRKQ;jRB6og&t$8*e|{CY>GAB3 zG{Mg&ZwO#ko(<bV^##{QjMf~Va7L*AQ*L*4&x|4N8LS&NCvR`z|#ASn{kYTsqwO&EK~ zzD9{?WGR*GgzP(^A=&qJlx>FW+x?o+zI{I5@BO>~`W@GCT}M}?%)H+-^E#jBc|M-P zIj4DFS4@trztpza*0$TMeO{R^baA{dLTqDsaab(l;Gq?`-E}SHo|jLI@qFO#bPGf~ z*3|~vyin##)Lr(v?zJ}lbckT0g5DOJdnB zdbvxfi7B2lC5sjNcfiNejO&%wS&n@B%tEqi+JN}yUTN`oVTeFGDJ1fH@nf>st zT*k7oW+I!7Ci4CTS`o;4F4Kx9D|{B?Be2btkrB~68o$~CQK)hbXgU3=WpsvVukArL z?Yf=!j8APe0PKwo3wsOc44n4fE&o=tAMty#FaTYAf z0Z%;oS1UHREV3svxHtm4AT-G1(Po(TjKpY=5dNum2Xq9$ok_;haMv^xhOUB#Y z=-$@;wN>s>3bTmdROJ=*RApR5V=L&Ts_M#;}0PTmZ(2~tJN`x zWxnbA$sZpC_7!BTF*0wIWYUQ4;*UR<&V&Lp^N zu(JJ8nSebvKPZ#lb9Z@rYxY|Z9vTyKGzDxo+1ywN_PN2{$<#w+9_ueh1sy95=W_2o z@yEWUf*H+d-3e>G;4TvE*{FKChW18|pX|`IG3PBl!|HPRs`gD!yYLa_)E(O z!nqzfU32l7&osi09>D|ro0k6(}aPFBwEg%|ez z$>IF+j)FVO!$9imt4jEuiUa!N%g<6E4meCU$Et#Trs)NPZv-iYb>JfehfkWhgU2?j zjnf2QzaC2aaeX?|b1YP(VJz&J`0{(j?h06Edj|U%Dq(x#=&NAi2_jG;VcO9m zn?c7NsoCKo*D}d!m?vy_?ih3>v9BE1X9?vat*Aj)bgrk%Sn6EaN^A>UCPC7jj>Fpu zfy#cd!XobidE`S*(~v&6ZeM_V>~?W25C1uP&ApI?D6TPG&4o6NQGW0seX z`2GXl3n5!xEqtfQiM@M5FQFc&>R=9D-3>mD6cKNPUgw&MKBc@?;f22n@}a@>la7k6 zx*IF=30c!OHH|X+@G|ZhZS)2k^{#8!Pj>Ctb>l((Z>A?csBVuOqaI)qD2g#T;>qTzA3GHhPiTU2%WN@p$!pHW!l+$6bF;z zUY7~u_bV>NBr61_m zFs5sbA8ql3ZEcD4_jD;3XX2X{-@d?XopN_ocP8g*-l}eeG+9aM_}xQLdJhX?`)1DV zT48RJ7ZZm)S8ECTMunZv+Mdj8pcX)98X-|dJQ-KML_Sr5QPHjSn)>sY4==8N7QIR) zY3j=u@i8&13TFFcMA495hv&6s5N&BbBd!PEuUDof0;U#+@2zl|q znVyA?-kj3dhv?9X?`)+>EMZ-)nt0P1^Koc_DjFiUcC}6&BwP+@oG76(wS+cl#8tT9 zpRf_703lWM`4p*Dufz?9+yYLRFFa%Ua^+4Hp_@g!efpEoAYH2_5%7s<<_ym`MP@{h zwzu?ALKbwTA6B6T88tuBB!@h4hWvT(h-+rcEv;s>I)w!7u$b(cwp5gI3yP?q4nKk; zse1VQE&RS_2JWD0cuDB;1@BQ)oV+i12M_BDSsUd|t1(O~x|0ufbxKyu%bVM(eH>}i zTspnstVmH9HQws!cA3KR;9Y&Pk<`hu)3A=i68;}-%P&TV7!~lsb(3B8H(K(nok|QF zCetms8v+C7(DAVI4=a-JL(J-myOF2sv_8+096TNt@`uE$H}xCFMcI4x>f1ghVxFJ% z4muuLyrWyFFD9LTa_68%BHjEOd}RJZHwu$pwWd(9%+h1c!&fg^!lM)kN$-)5RDML* zc3?la^qxjs%ENDc(nH?+e6?+{Pl!8O%|^+swpB|Uv1{F7)-;N)A6C2Jrc%%|K2O#8 z&{OaoQgE5$Vdm%Ij@J}ei&iBNb?8{#5x(J6;!4U0M!5$5^;g1$53X9))ow=g$wE;w zo8~eWix+skDWfchX>57fCMjbmvAO3WFnt~_s-7h(+YTy{!e9w2j!6tok za~c?MAqT%2aNz%{Zk$8~zS(JsZ|hPQUvnV=~7YGr=3|CwGg)o7;7^0e?q>+ zK+X*wV)3sQ)#*TM&DKEaOK7?wsi^6gNaZgt;ph<5=B-N3kbsAR_5(d7DzCvCJvb7&s%br&#}s2ot%_ue)OO)$Fe*D`%8 z-=?!FG-b{->R3C~^2qD??iw_c2!F@Ll&g8aHRl68Sa0*GvzkwyKBK zj!=gZ(a6 zpo`SteOS7Yb{dT9#XoUaL(JXSv!3g9+wX=3?{5j%^@)&iS$%jC*r`I;kY4ZUB(rzlOFnr5vgR2bwb#|eT6#v#L^&Nj3AO=yS6fM^SK{N3n>0HkF+r>f0yqiW&QsIiHKR$-|jh)GSba^7k5aZBF5?#Tv~%X&aX zlELSOD}Ck%ub=Zn`AZ^8;2+;lBN>Cp;h3XseAsCZvBZC8sGaco(NF_X+%~-R{i=G4 zaiORg`9zIpjk*wo!jv+a6O}7%H`*W#ttVrr$nFrPPj%9wgmzuk1u`^0Vo-E9ikx5P zm=zkt@yOzQp4$BJ{G*mStZoCZAV>#sYN23_Lx=-|r-fbad%1$}bKX@TnQ{4rbPA*r z#}n@3Tp6}+h1A?zk9zDIKlxzn&2z|vMh)z|Jxvp#TV02$*tHIF>xqostm~S759HP> z(`zZM_NZqZ*5@d-Kb1jPkU@^XN3V~8!3rGxj7(x@{pr9ZwfF`DR7bXm?$S%uRa#Ya!Q0c39izLg@N6W?l%higvr^X)fy) zl@f07MhzFi9b@|G-;JFInO+Uy?!%X4OGEkZh*p+8JpruQAi{|Fn7aiZLb$R(JHWi_ zk*6v{T)RYJK4G96J-^A22RjtCkyn`;FG7V?vfM=~l9IPM=cmB4x9dub7RZnyYkf4W zTCJ1nAcRr<%GXX4(_jkva|F+OaB=JK?KiSNd7Y}MeaMi}#LROb4_eiDjs!$gf%)Br zwQ_vssrspAr95e=bVX`~Uy>tn!TVy{$^hnbhNcN42&J4!+YY1 z-gMe=EpyLkO&j0l!TDnbDE<3;YKU`a>m!v3VFW*FO&`K5@w6qzmf0;tOVnBx2<;05 z%9perJPl(nmP94Y#I(qZoPfkM*JB5ht}>}U@IC!3=xFQa31tv7NlG4_v`*RD%-0jx&t4$>u)5NBe=|YxY?6&f z;l~HLn|t~bo<6)#)KSqlhu*p^jgLTHu;MQ6AulHm+f|fxd2atFi_qqCeM8iB9 zdt(ZGK3wba;TUhQOu75ELnSuii_^n>K`*Rq_M`(6nlMt-Yia9iabIT6cGxyzz#5vm zDu#Vil#hV;Yumb!xX^O>$gs#qc1lB5jUU|-QX3I_k(m)Ju-<5yk|g>`_-G~Jh_QX3FT~Rq}Y6N|Fx9X{a3>DEE){EF{3_2nli^7v98Kr>{#vGV?(dUup z(zV!7oa+TLf?7+eq>i8wsxrvw7;n=T<4D zhALqRg(1MqUPUr&j(ciX!ld@aiGPdmR=@lK%lC?f4z&L-W>Q>UOJv{s|_5P5? zo155+_VrK44ITQ%;M`iCClYPe^ozS-?%8krsweoeA954qewa13QXe!HcN!A|32Sfz zCH?o-AW8if1BNR%IG&AP+4=dtRw;qACGf?Kr0WV+HZ}#c z{N)-vbVvvKvzFQyNsi$S@+(-{(pr#6O^SW&eo(^^8`-Sb{kt9)Lfm(oRVSOVU8^vV z{;Yy?A^im%V@Q=g^sa5b>Ww>fh++iuYoRlnJ$ui#jzzL}&VtF>3_-Dqfw0A@H@2dM z^#d}>j$D}6$NIi$wJ?RLvGlAJ_@@u2UE)s>G3Hf~H+;S#HzEdaM8{;4X43X?)dm|* z&G)XLWc9Kk&@OEyX>U%)bCiziSBHIyyIG76vw#OsYi?9}UGyMzw&*l9LNngfl9%BI z2gBKZ+Sf!06BXUqE`X`9TxNr)LPF0I=Sl@H*0;HCA-86gsP;QoV=j{9`5K3RCMwcd znKrgjpCrL(rL$wov-Z&84H0m4CB0b9eO2bI&B6C=o($feMv5BIkbH6|w46`j;W8%(gJahLdOYHy9oIJ4SyE&Q7u}0v}B1?T< z_QOwk)sp+0>V&nuq=j32HdM~>6H@SKi@Lh%7$Ruf|30j`0iq^SE*^dOVZ%x{D0TTn zpU$s6I&^VOf!X)A2*dxR*M-#?Vp1(k!iK}-)ZRVEl*Z7r=lbz(Qv}k)OEY-)Rb};cJFbUO#TkD(> zx|5I4cUMhQkZDNL&o`AR7`pP__I|;4sli+PU+vDDHRc1wrU}uSlUXbztgi=?TI!eb z$4}*IOcJH#tWK#P3CO4C5-ptPvyrDE8oNE@NPx6&>Bjko|0s7RwXU*DU*oizWNtVS zmUq=&w3aA?aJSnT36lDB(t|ONY>omyM%H_#`iZ$slIZ!g&Mf7PH?c(6-Xa(m*s3J- ze0a(4xku>Thq_e!9+^jO^6Ha5zuj1;G4TlpV{|5;u!UL&I;0&rdZ;?~q133?>buXehYp>RIB*$gj#U%Gd9)XEG~H!z+KSThrz4XvB} zqN2uLa!R^{FfVlaG7&L}u7iv%qH5n(yM`+W;t5!c$X;`TzLID)yNnIiYPi>8gSD7G zFM@L_wP}m?UKi{hJ~;7i>3ff}j%;y_#hICTv@a2-LP>4+*(62R4NXqkxP=0LbseYZ zg@Vm3o8@Aat~>y1o{rY#A8F|_4HB-@GB{G1@%3==Ii?{3TPXCQ}4v`WB`zoU(Qcl&bs=#&%ton!o0Z#54^?mQdu>o;~XB zmE&YDx45yhaHTPM*#D}s8B-9=?XG9()nX%z$Yt}T%YRelJz2-UPASzSWEc!WRL?Kg zbOt^4Z0s@`{bPdihE|&vQ__TzAV3QK>MVM{%xg;yz5aMopwrI258n0i+oLF=ftnN> zUz`Im%={tHQz5PYlgmh@dhxU2+x4H)kiU40&p${McH~rsUE%*^g%udfT%_5eP!!Lj zgqZVXKS{FJzeqCi3`{PBhxG7p(YVIiYHM~$GCs27TP9fuN=t5?FI~J=-5pu27r7zN z&Z5t_9jzJd*N|CT%Mo_u3) z=J;Wnm8V*-YvFr74+EcL_fNyVT&zT~HB9AKe*d>M%*!nQ7H4JGZ;U7YE57*6By`JC zwOj(RqD^(~L3=P{%nzEa>%Igc?$W#OqqPM-31u%uL$%Q{8+)Bjfw1;TKM=$@YPR;eY5c4BA^N8fR%X8hGEA*Q|}@LLX&(q zTZq*eyOQZP>982OP-Mt$u{lox*?bvzxqYm1R~$ zfKBlYm-h>32%Z*(_Da!19T#HyRocF!3m}Y#Rm=tQ@OLZpsT!?ZR}l+d_(bTj+F)9d z=GmC#vgP?9DgCar*Kc(S_GmFRE$7oV<$;q+bLP%i)*d)n+GQzmN<||>bM^O&`DB-U z5%&pVNzJ3uX>?Y=H1Zloi_7JmVKHPxh(rs?HuoZ7*SxKQT^}i``G0HX6$;@U20WPM zSSQzPMNQqfJ=V`$n%kb9zozowIfe#5VD^8iAELTKZIVZGK=}rJjm8A09)uytj!8ru zv&)w&0WM%CC6X&Kyr61u?8WueDZ3sn;)m!mTk(PRQ0T|}Ib9DH8n{*X&tWf>aG77;B#KbjHM{`FFq0dkvo(j~(*L znSD`C8&#fevN>e4>NNUMHv#UlQFC3~GaR~c&5m&d|rNN=KjK_!y;S z$X$c~8*}fi0z*d^#nG#|gO6IsJG8*u;0W!xs%8I#;%4s4S1n?6(j=%tX@sj#di&o8KY0J2$#sS`uH{Yqla%R7n<)L z*Dtd4?lEnhA+ox$Ikjd*F6u3E71(q&y{AEg==J1%wplNWypXhHLop!cpA_&hmO96~&LA0!lpoh-FHx0bNE156D^ zX30rE7HB~m{9OaR@ct=;Glu_pan2qsHqq3nqNPB@Mh&OS9A;4x>=2Oiyp61POv1Gy zz>2SurGaWTokBVZCpR{ixoB6XB$V};^#UUDogsXuohAQyw`Cvkw%tkurYvrI#{P4^ z#N#&D<{2QrB;W7KFQR)m`9*XXE5G=!RctseOCLdqtZ2NAPY5nEo;yBDQ9`3_zL@Lo zUCSGrBue*=uy=Ki(v`{(U-Kurj^AZoAz5N|TmQ$@H{K4T?C9_|nB@~-uRy(0yc?*F}B=Ia5i2w&Tl3Mgon1=J7g9j8gH|{h_KI) zPt>GOj4l6dB3eSPnz}tWe)@I%R}*pdh=D6FF2s3>;(Cu7Qv?0=#m}}`q2|mDekgEV zAxSqax>MZ;ntia?6p*hKO+#9m-;6Dg;u6TOHF1ES-nG1(;bfdM*R;D`L63W5s4SuY z52^2SgG}{Apv*ACm1E)ceDQn3<-9vbc(=DdrDYo6$8*c8gR-I0n{>(;c0aF*p?5xZKkvm7$7ws@esP`|ugArd zUU7?N^Z6xBl8UHY$|jmCW&tiZmvq|kWp3!ken+mo$Tof}jRe_@KYNKRd$Y&*Y{pM3 z@_K5(@b`zbkWKz?<(t9AR}2q$BL8Y4KEFhkZQ^IgRpvxUdGPxS1bqIf9f?)t-V7^U zF4xm{Pm|dPCzH0tvy#MLCAezuAwgmwj_-Vk4DICr+K_}}MI0BcX>R6? zsc>)H5GcDTlVEFg1TCQt+7sZm76z7l2ty)jQtxzX@tw%brkdY#rj|r~&kK`^WY0In znMU*;6y~8UvsMX1y?Gq;=mNHoc+)Id?r33`Xtputl_ZbR*A8=!cY0Cb-841>$h9*n z`vVk=1MpQ8xXn4L*H0CR?h6;2IWPr24u$cNJyV^xp6#obh@nUgH3HP`YD zdP>NQL@(FIe20#YdyG6<3o5QSM~bpLIpCwoML)BjC3#Tx=-GI^q4$TtL+`)*dn(nr z`|`?gCmr*GRYSb{#tXC-na)sKKDIpK231YNK~gw8#6N()p|FE+messj zpSN*~`9Wk%MEOMY(-R98V13WY`!7lZ4Q@sC|INd@85ozeg_`#s)@W3B*Liymt6Ve~ z5cmg}XgMhOBKd3b`7}r?t&iRHZExf|?bmd4314@tK_x`hNVt%DAJfOIrm8t8LR^ee z1bYi)lop{NZ~A&3q*H+=zjoU1YH?p4PA%@3(bn4*9^HP3U@2`Lb!!%FV`Zz;@3;Q= zqK*QK%Ztwv9bG z$LvcA3HQ{jqh6Mn^tuugz2~(hm+^Ih*GG^&_Io;me&zXSh5wg4pHLS{mYfH^>Q@wR zZFi9hw7yGG+iPNC(5U$(H0gt=R?8)R89A-7>T94$w^*55=Yl0?fYj3qIN3=a{QXSs z)#c!PAMya77N19ERg*DJ;9PyJ+KGMb>enDWk2v37q6%QdWOB)lU?2}|C-%_wvqK9; z;MQz{-0DRRv;gXX{b!L8y(ZShEB&54@8d;VwNckfhvdxo+YG-}~}3G0K-Z3J$2}B-%0)1h%mK zgZK07+;`qjc85)J&j)LGt)rpkQxMXBvfBY2?W7z6NrMQe&)>(2f8Iz_SgSn{Y1XZqrPgFdVy7^q;pu^uoTD-Z2ktkq43=n9?-P9 z`vNEqE_&p>CL=>tcjm{01z3|r$gx6rxa0C)rG4`&Wv3*(oFQ&f>zY)4@1l zhOHvnZPNW5RNeo3P?dM^*4Ln_()t*^jawIH%ynwl$OCQz-`}k~t*DIRxi?Ppr%zOS zt6=4AQ{_($H^2nKhHbG4w=W`bx6Ah1&jDTcQ*agMaK$rQ@gp|Ic|O7CO!yGR(<3l( z_}rf1BG0nfvZDy`Whc1U@=!{IVXReTn(ztG5++kumwye(Cul{NRGgT`aQN*P`klxXiMD zD{ zaZ}pgu*nj6?rSSI+c-*)|6;!snY(fON`_Y8H~1Z{I;s?i8lJ7R$-S7A!U5P8Q*8?{ zzGB(QhAI)Ol~%hWqbuv zeI{4^JFWAEhs)|qSkxgQ?Xc${M+<4z5IA>h7~ynRu+$1IH}}~5Wg#BGVlri9YGARL zv$GX-l**^Dpb{>&ub`6nQJ)a+T?VUsg@$41`0KX!vY|q4Vf(g-Qo>LcAEu2Na0UZF zN2(t)uAWi1Uh5QS`X~_RJorFC_-DIv*HxM_Z;kR3HyvR2;NbQsf12>tW&qkOE6W>p z9LoufeAXkUhv0ZN_WEx-lM0!DJ7Ra7IdcYe$WV z1sF1@_ZA9Pa}bK_{c0}_qk+9-k$BX{eWTJMA5F!ob|8SmuP%c7@V+d9ET1AQEcB1P zG^RiI(hU7Aa8Cz+TiL!+>E8a$&`eN|S7JnH5@E5Dp`p$z5in0C_k1X-*uL=>dlEC; zS<(JdT5x_fN5C*l2qMR-uutU|0%&*zTByL}?YspiBG+c}FWLgT?GvXh;+$L8M|=pV z{|o?)za!-e|2W3$^QXuO`CQgXsO~Xn{UG@*1+6YBFjS z_Qv%QS*HE~@yL#r*qBb65F7gKG7Hth;)!czWxBJ<96-bCv*32I;GZOd|H|br5&_?F z0bIKxr@|3vBUJj>U5>1^mO9vS$`!8mC>}Co$CPpt|2;8j?oqKvq;&f<19IJ4Y8)#5 ztP!$Vd++H1!e@7h;_4f9&~S4E6~X)~(E63f^)>Fs0?6Ml1=*isdy#OP!PAOm?BPGf z_M~tA^T0n0ZP!)~MscFF%tUyaQ%>#$6cNdc%-;OTA!?%A*aV=@`7+YjTpKBBL=)}S z^P^|&WAgv&;GR$)T(0l_CMKVPayDixVdCR_c6e@HxwQtv$gLB;U|Ep(h(+duhhz$GIM8ph;q9dRr?p zY@TG$l?qBMYwKGNZ3ovwV;tO=BA;99_JnLSIE7no^^H)X<~&w)=Uk@ZoSTOB z2g61`*>Mu5zJ2X+0c$ti@iP>SCfZRDt}Ijkv)$yAT1RHhH455QoG0((rgWXjs2%iu zY%vfr_w+;zU%bw(n^=%ZM5$=B8$T6ax9@GRMnm;;`?%B0sN$fvm412LgBg!xL01hO>`4*n-F@MybBo z{!1PZi$X}561#v54a1sCJl(&VOZyD%(yy^j)l>G&qZg;tTq8ze_y6nEPrK*GQ-3(x zIo16Y@HYv?Ad0*R_g~sPHK89-5)H!Vf#R4y@MQ-~*eF`gs;!`JmADq-T5+_f+%cfg;UbDv>5jS$BS6f>?BaKPu{ z06j(uoS~UayinF9(@o%<{h9|f;ALuH3ENTRQVv$5V)V{le6(x(>97a+_zQF&?xsC_ zYstK8QsH7r-vVU2j?-^HM}b&814j=RuiTFqSxZC;@aVkxyuRo^im&4)dmG#O{^<{( z2}a?SY{Qj_k1~|JkU^Gvm=#dai0~n*+w{#J!UPp;d2K?)sBmKPTdWV~)pZ_0&KhWn zXIb13)#5-HRy`!V(tFt>Bc&5jpg)CWXgGyp!N|&n7qVIwp~6dL%<|zf1y}Z$J6wys z=u(y=@nSllB#nj-j?xQ)7RNzepJDCDa$*#8a z_S%)DSQkr=)}lxUB?S<`^5?WD;O$(b6OYz&d;9YXM*tc?nu0!Ucv1_9ov4h*wm7n;I zkyTi}qkFhPbL}v9-||+_u!h&(>GYmWJ$Ei8=}G!De&^4Txd(&tf7CHT_`{XW_y|*L zwad25@AX9)I^bQ$;2EXkQKId28rIFuv$J}H;nxhR<8JYr2Dt4fzr50#h;{0TIDBs; z9lJx=;^9<$72w8H@kp32hi1m~>bI9!6ArmV>s>R&k*0x9-)R{s>z8jCE|bu+&xC-i z8JGM4b3je+ig!9LA6cxK250lTbnw-K@rSYsB(7lx7;}$85F7`sOH1BV&yAVz!7>Zp zsnCuTQDu=%DuG|NPDeu^(tyyhN9yAjLWez;(2;{y0)&nPw^AE_E0=04qQf=sS42mQ z;#CN#$zRH1-37?QM`9^Q(lM@ES!@^e)mh40vwVr1`{Yvf77j8yOMBg-vZZ{FG zRQJNq#Z~i!flZEv5S*jM7dj+DZ_k0vutS}mGdzi|f2r+v8`0B()}n;Ofu#k}Nr%8Y zw5AYZF=|iyBK_x-k<%IlK|?{ojCVh#F3{V2w{!%|f3|dB-u1uiS6JmM`Q6ePmjITI zk~d1tU`91hE3MnhOi%@4 z^g^7WlOda*?N^G20-7s;OCo zTUM`3#St82nWlv?vd_n@Edu!)C?UZp*|rK_K<+MaqJW|(n`e0$`yb6eB0}oZrW%Jc`d1>%e@g;kliz3*>XITywDygfiKzmZST(Hp28k-{GnbU*ic1Dh^n24< z7_k9;L!UL&sinQBr5aX#5Z%9(P?$xJ+*$>;IQ%HmeOxNc@+e|M7}IaHveQn6C`L_# zDXIw3++Zb)@y*zdRvk=aez-`g6m#8kVIHv?|*REwz8iAFh+s(>FqmDdTU#^i+(nL4*Z#|D`09?FGWhHM>_-;fiQT80xlz z$0_SDRLroZfL;76a186h^gnvxboUe4%i_9^0Flm*hDeEy4UK*xY2vH6~Pk0pGX@=GqJQw-* zT@d>C^$9Ci<}CU*^#|_ERDUtmkp_S>@z7rE~i) zzL89QuB9qx#xZE8rFSmfu^8MUh`bxng2*Bi`T)u=cjS5H`$GYK_)YkUhH6?Ns52_j z5m-M?_BjX1<-}d@z&3R?u;W)(2=1kocAt11bADNT6MCkT;Z|#Ss!8+H%kIZzlfVlz zQqS>{3RIU3bu#%c$4XM+X#61CHIPWL~eH{5SBXd-$vG2}_7wwt;Z=U}DIV9Mjfib6*VnJA3H1_Dl9KD|28!-O0RA zVb+HYId(;a9&1_NmpWUgTn9pYIgKq>Ma*`IlYmz?uP$uC$&CQC&>x<-#x~e_vm@R? zA4kyTF=k|t(4!SRGz+q2b%ur9HA@0}J`sSRlVan0N^B^ufBcR$l_kAv3AS4$f}o$d zV^xv%7;Zg-vIhLbn~l+rtsrvcx2&4`q|IMUSH{lY625kB!aPm!;`?*xe)~y8?yC45 zv3K04wd1PR7XlzBKL(7?a1!>6I5FVephZfU94Ilz_KJ46@n1gKYs%`?#{Bn z*xj=0V57Fq@%-gtTmkTFfD-%<@GA~f!sgvji74Ij<`M8G;Dlz<|3t7_)WKB6v}T>K znM>|Z|KG1q*tLKZ*dqHR60y-#aMO*uzs9@ozOj7lzulL|Wyz#uiK>koqJ6Ak?&?}x z8nM>l(gx4y!!wvZcWnv}9j$KQQ>=DK1S{UT}`L+{|JHl?rWNOgdx+6*@5 z%DDp-P9Fo3C`+xi1Gn+uLhcJ8-RwXb`jdIt$8!M*!T0>?|g(u)aCeVv1-ax|F1+6W~MJh5}WFim+UEKs^*^`V74IE;36_1M$U49 zXk-%{&rY{{+1;3|D=DN3z#yt%U1SK3M@pc7OG0-thI`qbRx>bq{oJ4}cAr6F57Up^ z48+zcs9sV(l?dcqghxI=PD3VQdb>IBEMXIf|sxe1+mUeBxUGpQXpDD<16)L8c)064<^1DrdR zi<+*@MJ>?bz)28J>wog%NoJXUXFjkz{A_bBVF+5f0(=ZtAP&#HFss z(;?3lElKE7?&bFkOlif@e%U20UiC0%Qgr5Y%s`C=RwFObmHyxxCaYSP?%jO_b_pVU zHui%!;4s^lHdaC~u8n2Q0@_$I*(%&F2 zgx|QmIYVke4a7?Er&Ly)d**4j>z*BYhIP;6_fdW!ItsS@4BekR6aI6o&(1bQe&)AW z-**itUv5B!un_rK0bUFM3Q+w(%aNHra-FoNL`|TW^)~4EIRhI{G@nS+Fi7|1HP-px z_V~4&HC6GYoCPL6G8Psf`daq%nr!H?j94WF`i}0~nld$Q!OhV3Ch@ASeNB~G|D-z_m5^)S?n$lpRvQ@N5qE& zspAb!fcOxSLKv2dkn23!{$70;q$^7 zys=rYp1IvO$aaoJJLK9!yJi6h_fw!wjqpdz;~#M2RK3ftM#@F6{{5XjQbzh!Fwr;T`tnYJbOnhk});wD|aTVG9@Yix*(QXY@1 z*o##xjQLgG#IyI6RAA)|KS90iIKlSAQnKZAS&pKS4kWByp8s&o5uiHf+H6l++t4P*+R@%h!xmP!JVEH-*YZ?~2ee>5O?NAKIs?~B$dHB-IR)uvky0{m zosA3I=g(3`t0l7jR(>bi;sL+36r#byKS8d(*XQ66^~ev9#cTz}`*q`3hW-vya9UgV zNTd~%Z;e^!)%`;8x3^bsO3mki_=ITXaep7;$*5}t#X*O!`Iu1qNC(wTri*-mtC*;U z26yl?K6Ja&Y@3pv>D^9GqyYA>`aB&hdAfE4L13bN%w{T!ya5 z%{wE@a!q!?YE*1nsL0|FlJskyZ5dVo#KX&59K6C|$+|hhle5S3KjiD$X&N)tUOG}r zavsN)sv8mG>?47$5CL=GDts=MmZt@48?9D@6Ecd2M-sMls1KLz6>g52#5)vVDBD_W zdKAJgADKrsikAJYTXnDLTenIB)6dusXF~TEhcOHshnkF&((k(siX}%sX!qb{P~x(R zeOk)BvO0Rr7bLxbOp&KBxko58^64o*z5Ff{{sh#i{uuDfY61aNhMSv=%Gz&?0?HilY1HJx3|&eNY+4i?pA=Be$l;{WU*|o)m{l+}lqDY-bv|ZJ7%= z55sTLib1>Td)Eqk2G`(1xUQQG7Y%KU(qc4j%>M{awvV&EaBpU9d3 zv9Y;9_Sj>-61n{`6bXz%LuIQv`t{Er+tm(RE<(eVN_phpkF|19RoOhg!5WF+F;k8q zrJ=xBJ-oCF2$LNx;_-}y&xL0cTg#yPFPm!R*iUosK1ooKoBt0?GyqU~43-@NOQ}k& zNL9!A!rhC#0`NJB`4e%~LyUvA)qcZb_C=O6FT+#XhgHCnR+O~Tv-isKY@gHZXDqJo zQSTmCv-PtS?wuk|kZ@hzTw81)wC?nzWK|p~Ex1FyEb{vG>#R`0?8me=9=f;9U(C+V zE}xrJoxT9Jvf>A%ACOPPuh29OB%L!*8mWh}RdU}JzA~v&B3Y@|b<*W_duNyNowyea zFcp>-!kwxkk&(!R31+>P4<>|Bc)p+t2zCjI$}TPNlfacFzp+R zI~aGyowm~O?O}vmx#vvskocRWDnN{fc#QY$_Qc$_!^VaRXSY2IClKYuYu!C@VQx#q z1G;6HcRbkL_8vbVUgjmxFRE80!G7YxmKYwg+)L2cj}}!oj$7(RVq7EX-mv_>zYQ?^ zr5^kzEJoS;pOcDv^*~Z_!Zp|VvsOvdYxnY#-PHa9KVvblYuxxxby`=<0!7v^#<`Pk zB9=`G^fj)_hnvq*t=<&pobBM z{lm<|k{^@r{z86S8|=GnU*B{lT6!qowm%Gn=`{;mXvn} zf2D*eD}jeq(KJtd$f^KbpmP}Ro=UxtMSZ36Gt*ntAxQA z4mICe>9xq@+c)1LG6aQ9HbYA$^J8n$zo2G7BOh3Il-i^&@6kNznzc5ywd7Twudk(x z6JIyjU5)-*OA4NU+Y_yJ{4QC^86H|SDkt@$qgsp_n%I;+*8_g@ z8QuilNI5_hKBm$8yzW_Xckk*si$88g-}UaqTAr<%k3CV1#(Qr9A11*yPJ|KSjx%a! zhq7oD)9I1=tm5=Xf^SDWdLD%%ECFl&=#P8Rn_~<~mG3uFcy-hTz5dI3vkl`x+*WkY z8=214(obt7&}P`Z@k@f;SF`Wb!@D-&VaH-dpJ(fJ9ha3)LIr)<9Y1?BHvWO>2m8_} z^0Hj8)=O9uF~RJ}UkB{Cw`{Mp08#K#nVPOoe*aZnyRj?yqyYUnCgOvQ!8$*kNeJxQ z#;%7aePjxN{B`JccJ!~XM$=al22t-#glkpfTH-Ar&S-U=uWRGfHb+2Qp^p6!X0UgU zPa@2MpIzn4n6^kBYIZr@;6JY``<}*u(YV97ap$0YHKC*$6}O?cP@H+G{tU zMC&x?^J-n`+skc7&K=%7krl?IBaN}#&*gP|?;JXlkvMp#Rv+x)C69Ao>QG%90~J)T zX5xUOov;MNPhpsLGsD-^7Yk&vzK+q#i9Tf_#J|!+4m}-a)hvD^-|kf4%H-p!@w5e? z>S__@hTcAYhFS9>k8$OVWI!6yEVr6VOgU4JpjvZpC#7R(6w4i9$k-3_p1)h z^>?hiF5}$_vD$y+(5mn7zWoi#n=!Em1CR2b3U4%wRAi0aC@O-P%r&2PTGAOvOPkJi zoc;7^ZeXB%$i(wqPbp)*g=)HX@uy@_?-+V~$rw2K#9O4U}l$9IFU2pM@=C0lRMzWK<`T)x32 z`X(}JJC{c*F?H&rx*~kxSwi>DDX~+Z2o8EBmckMi_rT2f`g-TsjZq?Yds{z-SW-0P zgAJ7J_Nz~W4ULdfhvr7vbCuDu&usD@FOwjcTI>9`rq5L^?9awjhM=a7?D5QdcaK?E z&i}A-3fEoK>g|OMLw32o;u6~Za5C>ZDDz;{s*k|d(z}fR!`oR$Mb))`A4KVrlvG5J zM!E(RkZ|a3kOqMvh8R*n0cq(TX{4oup{08W>CT}W5%WE$cRbI1-_P&)y=%SiTK=Id z=L~0`efGXS*Y|r}`*Gxo{Wu+h8;2-?lOST4Whj3Pm#%$&aPzLtO$6H(U37|a{JYSt zd&beWh1(b)_CnLB8+#7IT6=TbozPFdp>k-^vGh;sybLH^%J}(W3xbzkjBG8``!Qh* zW2|Y@7Br5Nt4+%+zZkn}GD6FIzdGE=mgPzVLBc*SX9qPU$ilvUko_XhusF@gm{lbV zo{%LMEk>@cS}oN*ws>RlRdu)Fb*B7>{tANSgWj(!o3HT^xPk7DU6W^}ew9FVlpbpW z<|Da{+^0Sa~$nCA{1RKft9_;}%R0%%>0l@pafFP0 zz7p>Ky?!|@(po+JN>z~XdfssHqdcjK*Q2S@ZB?{=ef(8w#ttCTAd!<@v_VW5;kFQb z#}=i{M~f@@_+|y)SYFg?TE_x%=ZPFn*TwZmh{!N(5z=X!d{^`e(O*)AUL#MKFDoNlm$w40pG%X(`8W2|YWOPwjk zl2F;d1JbJR-@#uI5FGO&Pn$epcP)ju2i57a9DDV#_e4a+UubQf_6)`1M0>J6Nh%?l zZaml9fDW!GcSAR78rNmV6(6-j!G2qa^hA?|+Gip@2_84Z)_H_-Ow~eF@e|^aaMq34 zci`61D-+(OqoZ9l?a9ZBiY#`7k=2t*T0OVBWS^Ewl`|*cc4nVejlwON*hvRuJ!^W& zisM}qFb#HtrqPg+U%Q}9aOyU(1GbnN#tbtR(H*umEjd5;U6yVxw?Nc@aT)VQRY$Yi za}>+fDoW0)AS}sNC}S`RO!JuW@QF1@=L{-zRdP7<&e)Lbhn#~d*^NE4&Na7z_Z9bO zdU9Q|^q6nG4uK7=o8e$Rba%B(654VqG6^zVdr@*^vS79%!t6V-6_!MsF`7qo>*vK= zp?`mcM!0uwC`w|k6^gRInA;wNT04HhUqnE{T%6}9JZjvk ztk8!8m3W0%=t$tfw?c^zX%7vFj?$+g0|HrTOhRj+p*S&G2CA;&ZC} z!=`S3a)_g%lG2E1(4!s5as7E`g8sz2XIQJGDJ}Zv)JP^8e~1wonE=|yM3GLghqLmC ztPpb_*AsiyRE6VI^kc4Oim!RZUNeLPX(l=75Et7NMcO~6xnSOtG?YG+j z9w#+?N1DdbzqEU6%eKr}T{Kyl{R?`E-Rp!yTCL{7ESVk6oxHQ;wLJ5>X78#e+%ya>Zvl8@@eCX{-nMA^KA>*9G$jXwF?eWzsqTl?xk1i&I1-oBv7eeOa%@dZy<+4V;s{8wBxZvfTYZV4Pj{LClfdu>! z?8wY5-Ni-l0Ow^?E^s)eI`-Im59g|m9jIEp!i4mo8fGPrZJw@U&xXQDv7`F!n<)ts zIo28Ey+UL3C-SU%wtKi#G$_XI0Jx2DO!9CH*;Ds%3ntZhc@X8R&pi_6^ zX9b!ofsclBrAlLd(HYs4^Y3A2R26ten=1uAJv3wmlp?LVE@*e5l0|s$^iSSX* zy^B8Vo&g2jQ0+KykeS*iJ*~kmFLY-vg`o|BH2(ZWjVkoZm*(=-y8_36_lKfhf092F zyzqAvxYk6@TQ+ixGJ{Y*vWzK;{Zg86|02~*%$XuHRp>y#u6lH@v-H*|(?@;7JV82( zI}gh75_IO)l|*Nf&mKe6G)u3ZxKR;;L5v3q%?e{)TRj@I)D2g9lb|9Bz44>SReQ8K z<_c;eEnwVK16Kq^=venCER$++$PoRxd=&5;*PMpT*Rrv<=DnH(%TYS7ng*m|xCg-p zId_L7)FBUV4?XV&F;A&#dwB#!D7A_&UeB;0EbgH3bS%qSAp=o@3B+5ZF1MJhS`12f zyv-N3tjxN#wPY5aT*f?LVKx(`y4<*d@hhtD#?4+IR`k- z=6nv#%%zm#$6nfAH^tp?cuK>g@DWz=Z}$|+$=fv=^|t2oZlv8HDcAMv4v~?_onJ+$ zK6!qufLyQsGP+2ZeoGPS83|ErBYNcO5rchTyDgK$| zRf=tP>S$EsJo{~o(|k;s2=)feNsw+&Q+W|gxjChKRfKP3<$h7flXh_aCg))zCNZw+ z6qb-?$FkbuxHx>(y?jzeIe3w4&~koCO3);3BAou>V@dT;rKWBD!fwVvd`Y!?=TsIY zvI3PP_SiD|Fg^Bo;S1Z~`tvWNcp=0#svO|0wovMc3HuTbn zgN8)A_L(`S92%~&#CXB3=R`4z4xR04hpT1SRV)KeNDYxjdeAZ|f35_1)+t#7D*2dxh8o|>pCuYOu=W*;^fEeMah z8>D6H9z-7*nKyQGnw(dV4(W`MfO|EC`0Zy_lh2_jlH+1Z7}wfparck=BVn^w=w}hn zX}T73%3h7hf!Dj2x5;VD3%J6gP_gU)By6+7SI;v-L%mG}wYGFsE+bT6lfAWspupd0 zH5h1Pt84FELZo=EQg@O|S@W-$@0xqr(1S#T97lw4iToj;qq!g*puYS=aDPklGEfR_ z!g)BCa}7v>H->Y7&L3bPGmxNA#F`%k>dY!2-f@1UPH??D`<&PX5a&#{dx1-~5M zDUC(nM9)_uhzUl)20o5bl#4xRcwYGNVRBdgR3qFZ%g(Ju!DI9FdxXH%&;j0DAW6ps zqU-~EBuNN74@8xq&40_%@|3#*$6HT=_uF3Nwa`hyln>_EGTb?3x>TvwBoy*>nJs?9 zEZo~*CgqgmWw}ys#Z``@^0Z{RsZ{wa<{$6I1nnqEH-GHE??;WWp`DfqPI$OP9%+qC zeV3lRW5m(iTLTluK0(d?#u~-Jo?~BQu7z(+T`qf)`dsxN8YqoUPGu%RXQul3ujcCa zob|Dh4$Yrh^P_#BV_J(-uyJ!r2 z+KzMHAG$u@uU^)$E_-EEv%jB!ns@F>RVmT^TA_iQsrfQM*RMVQ#=^{M?;2m9UG#n7 zF_VK`+zFN;?fIC`v4_GP>a6$x%ZX-=@XQrN__-86Gq|;Il)UwjDm|HS^(;BDNTRhW zmU`pG$U?Bj_)w|w<+=vDddRYs6?2!038{ z5m}V5(jobv>9KUfR6_9WH?uDd!|{XLlFDSYr98W@5mBW23)|Q(i ztI9!S@wpa?Eo}h7#l$9j;wrD$f+=^IC8_gEu=A~mFtx-RD)*NU&j$E5=cZ+?zt9mC zxau*n8Xv!=e!19vP&BQ67*OYkl^3C2mPhswl;)k=2&ZcY`1@mB6Z4jGp8t)CS_^@)*g z!y-@LH)6*OVZ3^FZg{g1^;)#^mF_iC-Mt4?QO^mhwv9F2R_iOZ@YWDukq1@xW9$pO z=yghzUYZKZz=|}+uD7e(Dvaaat9(?RE9-tanuwS<_DcHpIJM)@iM!5@`J01kgdr{4 z4Ur#;Ff*}Ucf8?$;GxQ9nKXHQ)Sj^S2qL97bj#SBuT-bP6WA1i;nR5$9(23BpfPY@ zQ$EEcNI80!BjqqTDcjuN&$cN z9f-_<$k7(V{;Kg_1ggkyKN#SLcnipMkKd{9L{B4KW^yA}hAEmAhMp!DfB#hgHGahO zEgPng#{8Q_Q7qQT6Jp`Ab{qgfsi!c2KnDX}9I;k6d+nSmYeGg5rHUPxDf*&8D6Db- z6A$P+vtHxz@P0DE8o5Vljp%~Zukf96?n!;UK!NHWsQEBO%tzSkllC~{&=d}^ierH$ zyD5TB5l-I24~VZyE^(+9@(vl}Rt5YPS$!5Ll`F>CQnb=p63K{8$P^(m58q=4z6+!S zd|DFt%#S}mFcfiUh(&oqM24!LB!BiUh*&T8Zn{9bbP$zA0I+LbMr%JI>E}|c)(Qok z7rpBks6cJ3KR9?_CMiXkj+c#J>1h?{(A6g8kz7yyXR!K#QZ(A2NrqSK8Ip^0?V>1X z22s#*Psmtf{GezW#z6-(6Ck3?xaQ|C^#fq$R96tGtc7t;FvbIhE~6bco9-w&e#WWI zaggj;^DXRX=}MOC#yrpY0}g#8rA-C-V_xXFGn`#g8u0aHRz>^RSd##miLQ!N%_rMQ z2`NB&NJ&YnkA}0P8MXI%yiI_HbOTeTxPz5R_Gsq34kkX8YlGzya?RhO+p&SOWZk?x z7UlJhKQM%UW2y_-a3JzV9 z*GPe_ubixBB^+5Nw9 zR(ij^;Av6>19MXE+t3%idKijf7AF|4n;23qM$`!_3PNg*lJWK-WY{eT`D&6AY9+sU zi}#sqYb`W@vRa-rpmOhW1TTYgAWoMhK%Qkq3bjtGo}Thg3bWEF z7Lb4Q9}svChDEUrzaP$aD>@tXQam2zFD9U z2#1y=vozR>=4sTo0ZCPno06;GbFg57wZM8j_X~vV!`|UF8lacCC?p#4*MS< z4*cT4ul*z%g)7(%D|-wj08KuIvcV0+&ek#EpPev}rPIe3grM4QaqN@(fXv^8@F>ys zwUY&P@9ds*&X<%>8@$H7@#z+aNvv%HIkkX7U9j9;aK?DaG7c|vr;tQ?hp&?AaW2M{ z|65G6oA`-ogYGi=T8aE;wH`w9BX(w2-Gxb|Ed!M+rxGR@<(e!Md9Sp_?Mo<$ zQI#MpGk&9QNnEc?9i9PVwS!17M5CxzB&=v_}0HzXKGN4o4KoWrw=%<)W;mNPr} zj+@sS_)C+bK%H4fwXV+lFRS7eS-*mYFk2l+ijP#s(c;mVSRip@F{2PQch9s3F%5=t zMQer=rIU6iUVdlO{TyT$qa~*jUWVFl6tK&?mzJslMAHTG0z&;)>4LEF9!fGn1DCbBhdJ zbLRcAU)$#E!sTK_q?}iO5t|D|)ve>1tlOlbcR?lz#zOPJ5~ImR7Op-7zDE-U??|jt z=P015I!d&UacpY7OmP3q$Nk9oD1)pCF$Xq_s+pC}6l}GRYRwL?Ov?(6Ya#v4{xocm zo(0_7WBeD`1J*K_H(cCFi3p1)xU^73u&B7G74sF5ugwE$Mw=a@aAq$DVP1_!PNyop z(LD6$SK{$k5*vFZ;*InUX$z98*-2bP7N!)_${+2>L^r|{7UZIhq!f3u85QWBjsjT# zcpG}^#cLFAUU0XI#3o2}@zOmSB~^I6k6S!x>{#Kcm{Z_A*QMf z%j$&(D!&|(jjO+NS`KYR@Pu;DKR}uVnjb=mDQP+UB-DMiaui2tPy_>t#}0i=6_QF8`>pFz6OL% z_3oSQ5H}CEXDAy=qCtqjtD=p{-}sE*qBFu{K@4O)auc{as=?elASVj|u`t=L-R5JH zB|Ok21tq;SX=;TsBpHhbgZr$J@v5c<`m$S84&)ER=-6DmS-rxOBFfVORuOBLcjN`s zwfS0`tw094IHyMs4gZX24pno@bDR^*af?7SwE7%Xf_Uhu$yn%@LZ{N`OEZtD_4;Pm zq(abBeUk)sH^|vrO-2k32k}16e%G=@@a#Rk!*mJJ2m>1TuMcAZeTl!0!Q#xT*aFSV z-&R99cUgpw;YQ_|29IFp#@QEKqvFp-_0n>AL!}Jz7Ksc-Z})&>cH=x)5W5s) z>Q~}dM0WHc--nAD8PSnJzo#G=Km{I<1m%Xr+vczFU0yM1l#sMHt|#++o3{(Xz1{gA z#Vgx8^T|rDEfm_J;sU>?`DPZJGK7a3I_Jt8}k zaUPN8|InY(;S};kmqRiawi;h6y35kQLhGEm#Gcx|7UEvPP9XERHmk?*Llho8ui*Sf z!WZ?j@jFYv?xXqh}tq|xGIB0#eKR7X7* z2{#V47EE{JyXPW;Wtc8ArhrtJ?1;sVu7Ia|Py>WC79l+AEh(xKwA8lcX+pby(v&e;B(xTKz^-5O9rf+G#?;n1$lj!p^28N)J3kH8w*L7rqV5{uX_z+vckKNE(L?C=wj&YHK6cSSDaRkgB5 zP;MfUA4HOSjt*3WpX%k{~N{M7eK^DbBC z^)H#?H=_8d!jxfU7-X4tLuVnrku0;SoF@~!0*w9Vwg}kg^VI$tEN&nb5xJgPaNlQ8 zBhvCyZ|N+3LD1pcMlq57rye08qqE3J9dY;N1e2Df<8nyd?#KzZFYN1k>79If&_OnL zdczsi?(MEYlkLGY<-&Ou!7C8->u!;ohf2ElXDU013bV6rWpTU7V?GZK1Go0r(3Bd= zfu%jd-yrFSmIN@#A^h1n?Jq?6O<@7@uxHniRQ6>wp!1&k!VnQWtDd~#L9=`pcx?yo48h#=^t<1u=d&S2 zXf@BRt*wb@g&&M$rU|s*@F^~PN29aP0f+pgX1n3WP0X*k9fY&jlsv;C>blGOv}Rub zt{{juU(dqw1nrO$8kVrXlAaw2QH(cwKb6U!v?hR2ZJ0RPn79(hF}onUz|*NMtP zr`|0jB^7>#%|5l}6QH*pDbf=UNd5EyG5Y3DvXQzw>K)R?&G|0YM5x^|1)~FMnDkgJ zU*m~~x!@OS%h^m0E3v|YUzz{+HXuTg*xiq-MnQSWu`F~uJyEGQZWM;KB2liZ?vq;$m@ zojAYnU}d6Y5|!pCuGwyojXP*)2z6r~qrSTCU5+AM}if7Co3zoE~KOEj-F2e2vQplr< zrfk>0zhMXFfh;f&WbB)=+5sX2FM5$=CfEoR{)J`C!&<1pf$s-@!QnP8vgsrpx_Kb> z*70W6Q0dj>M%d303Q# zS_?zn{~b*;*aaafvvM z>WtbhDaT2W6f+qW(OHJ>;Tj4mu}7DZluYaYh&W7lyB~%!(6KhH2|w%Tl@A4?`MeY} zPkNb+7qw0-oH;HjuW$dO6eYsm=~B-_gPXGQpwN{0@%+Hg1bt@N;oNL_Llo^M#lTP{ z&Aj{)!W~Py3d+7jdl}K*f%9QuSh;HH$K`*@<1hYbP;vh3-Xy?{wd=yI1mr>%j@M#r zQ+=*(RNcnF7ikE^3emDlDuQYO{T<|vc#hKjOg3I~xtdu}_N)G9=$Yc6;&KvuQ|(FW zXrDhgnR}KkXN&|CylyUre7BL9JbGH&XwSwH!;Qw5*HZ^LHV8+2IKc!GUHo^9D~_Wa zc(ZQ0Qp54`WLa8HfgCRP*S%Jpw&8&Cp(G8CFpo#nqlkPv2Nobq?zR9k-Je!nCf1O( z84mn|-rKU0IS>Hd2Qw=;ce8gKi#zMcc$t-k8I=6kq)eV^EXUWMI7#uczb*k71eMYFU};G|r%9 zPVgkETk&q+Qitn}^6qNQGjw>u`+IT`A3oF;%^=3ZQKz3tqS5GgMczA|z2#anw6(gU zLh$Z-&X|&BjMaaHB8I9nc^!Nze-FdMmla!rFsIdGfe8%9s(ZnM1$Bo@eECQ9Qm+7R)biDlZ$25vC>P-`T8j?VIs$-H z#uS`~UjIy;wV^TUWf^+1tc}2HL#+CjXKcTw+%B$jh3eW_yv|E1I@Xio;jNeqCQ^Fn z*AcvH^1*lM=u@yB^)f@rMrFq&+(6v~`czEfqxQfTGyPm)4`$!dHus4|WE>}_jl6mN z@N#jrMv^r-4PV02sA6^X7a`KVRB6UNHx2U27ZoEW#Xk*=EMH5it@TPq#5|?6s(1*} z?#c-oE&smC07vYtMu}|);zi9=+rFFkFn~oR3Nngq+@OBRC7fet!|izg+s6;aGwOzA zU&lZdQ~j%p0?V1CE}ba`fQM}fK?2g5L8A%K5nDwRy=q{<4#Z?iN$07&>HA86*Aa__ zxm+RS-m~ck6bX2Pyy%yPjKg&1V-09wx(GI17QgBj-6%<9GuQJYv*0gWqVHQG-de&SYk zwarNOr3=FJ0XuLzzzrq$D7?a}#Ui+*m!T^OY^%F@_h_&$>Fc*#|8GS06Hb2S|Nr!} zOJw`Ar~~}tFAh53XTQ(}VLBW|CuSr2W~A6NV%t}em_Ecp?RKLXw(XVWi@@MEdz;(W zt9GTfe+(TEB3ydaCZ$2vZ)u=Pn^wI9o@Qo&g7l)UHXTKDo(FczZ9!CN##wqFVhJ8A z=I@28BsN}*qk1O4JKsPD$yF&Ia6QcD=>^iEHH*k!vnK1(H40Nbho&MSZ6l79!|?d z5`dwPjCb_I&@CKC)(6W+(1Z~5P^KhS6&&vhnGut|on#{nbPDn8T7N>L+jl1O)L5JA zg+@LOWIw1|x;+9Ou;agQi&J}6qeweu>8JGKK=Tg zquS@#=C*A(lm17El_bd66GXQL0gzG+`c_6t(=R%;#y; zRf@0gpuN+%S9^Mgl<**x>VZl7D{l$z#*mX*QaThgWX|HdSNi(f0@C6k<1fgk#`Q}A{@2MTI`;$B*iYyiSU0tdsDgEV{29jB4S5^!Y zyBP-UI&ks_`^F=>*PkML$OhURwtnbWLk6vAM5e4nBXzz zOpAH8P47d(rE~hkl*J`!2a-wS z4_6I4^KvjomivK2eQYmXZqA}{E{vF11>zeqE*anwspy+~qek%(K5}H}P+g{l9&acC zdBGCkbKATi?*n{pxJ`5)r?c+Hn=p})C8g7nG6lQlp2p^ADBfjc0=Bm#E8P3OM{n=E zNR;N=GPR}B>V0_OCe8Yw_R&_ajHPXUvJEq^j-A$TVIs$~EYSzw$rIK;<)49dnd|Nkz;aRhSHRNv7r-(N`yT--mrv)lAZ-fk0!7_4$JQvC6gn8w zd8f_dW?vZxEJ{OhhCB?Nh2V|+-?;!e+`c^W9$2+p=RNn@K-)^34wYSG3hrRnUZ_xF z#@9#={@=<<&X;thC7{-v9wAPNp{VpMzT$$>BnQZu{-&nzL>^0LfM~q8k6k z?KfwbxO^?>t^CIYT#PS%c%gUEkQxB zc@A2$zSsPumMY@kBGf{w{^Ir{PU+u&fQ$1tkKRV{D4n=|W<`WvU&c#XzP7o3S;P)T zJr)waGC(JOqqY-9Rfn$K2~DthtO8igotg3Ad>F2K!$6J8{x&9Zgo{2^ybEm-OA)WJd_sVpy;Il87Cb8vVF@k%?6|spFz-tt9z&NOpv&{a-A3@BceX9<01pxd*M} z9{DTmu_>~jRV4c!|62NVfQe*9*ssvQRi+28*y*sstER$c?^m-O$hShXe!7xQYo)P~ zTNwsuFjHGH^O8GCmTd$zzMvR@f^1!^|6*N_&HuhI$G@><@W=Nc+?&zhz;XyhJU%w4U*t zX_-)Xv=tk$!1>MOnZt%J6W}J0G|z7Hd{Rp%Gtr1ibN!-EXh`srWJX!<*$-M`wO*Gw zla=eO=FdZ@%N8br(%Av4-HEMK=+NEX1`=s(mc$`eC79zaZo0u-{#9a<|}*oN)&z zR^pR>mEM5u2yu{=h9hDgPFhu=C1D+tmwuezlw`@AO%dy?dx7Sh%&2{Pkr|vCJCT_h z9(aum=;%&vVDnyT)m!SjLpX3Kmry(`sz~F@HG_Kr8{#@|jiPZ_6PH({ zOcej}fmX41bV|icY8Xpex#lXCOBTqvCn~}Q&g8hsG8G@n%i2+y+9XAx>Ll57a^+H$ z00Z9I4@>B}uUK-sj%P@P20fIInj+yxMszpDjDk@6N+QLZ1v^$|@Y}n-E>wE5%9E`o z5_hsnMEFOIg|ryLZ_s?L!EH*tbSaD?!QP3T)J<`BH3iakVP$fE?8N+G%}@f`30X@z z`Jo4H+Xo*W5p-G6j=DeJ(Y2#9r=3T<(%I`LV1=_g_(uY*2#*ZMk@JAs^NggnZ1wZV zxgO(*yT93Sz3ZA-GJ%B}Vwi*Ur|u*(Q|E18+_x`~jz3U0qr!OTYb_R;a4JvA5hQKw8GmNXz0DK_FJRLE?dl zK?Qp`54!Q8D=R~8%HT?d!Ph7kWt8?ck|^vy%6$Ni$%L;W64-U<`82b!QCIHsoyZX) zm9F>`NPpM8c^ z&M;U)DbCh=ao86lSz8Ap)I&1Hs5{Fg} zH_705GJp*9rb=g^-f~k!?_$4zj zoZE(FAefnir=KgA{zvry0TVn2UfR?F&7d<5Z;C)g(4FvoBb_l7kk02b=$;L5HJ-%j zASJ+tK{gNnpA{(?UON!Mh#_y$_b)q=j|Dovz)H zi=?!pQ8vy*h)Uo!Sf>0{{Ah{dy|i8TDY2@;WxQV|!6&f&Aw0){`vMQcgFFw@(Pw77 z9sDLf4q{O-cnD7eHw(gA?}C;)_I5xOH!D{W64ghTQ(ozC_y+aeyk}e0Z8u*cg7tBb zYDOuBk8LuZG*#2bHM-(YdaQqj-g&c01XmbhW_qCP9?U#vk$07QzEkPC@@;mYGHXWJ zQ}P|yKfOP_H2r`>G?q8pV*(E}DOOS8%r%A~e=f6=hvfaf1~=p&9mW8XNuCbUtT|+s z+%H%luD7qrR0j>h+qeam)+Ps?^fu6r3|ZL62ACV@J`DYev#8BL9!FF(*vRhua&mw ziJ+Hueu|Q@*zRweJ%0x*s=j6d2kBPV!^&WdbM*?mm=m5eLq9$XyK}uZXsPt2b7bOu zO6ZvUc8sRbx?|}Vqsfxl5z)!u!i_=iLF!UhZ)f%wt~kvUI)!;~2h2^kETDw5{gE6^ z)kOzqSWy(`U&SY5sbe*xKZ{TGet#05_@sY|PcC$YQxZ&XY1?lcVEm;9s@w8f!JU6| z*sbeP;CCaK5!_4ggBNZ5AzCYujY5;E&bWYt5)70){IbF0sd!0UXh}&rKY=R7-x!S< zRftmN$%u#tBm|RG)SJ{*EN7rs5J=g-@EF>FDG6(-G|=-eDFKy{l;&mD`wIa zs+ykzP;*=*IIuGCE#gB%hof|n8~1XGM^pK=u=4ae7sHws9TD)p#d@V~blcZ;4BqT7 z6vx&W@4X_YdfYRu1QIUMZ+P!oydt738V0nCJN?Wn=axmv=sgwmq&+5O#UUVVFqVtp z7FnhRaqXJYhpgScK$LK2?x*hZbiyZ4Db|ZZ;Gb(c1oU*kqb8D1J8cQn+&t< zcJbdBD+2YW%PA(^xax#+>lYDi>|qvPBIBP+qXw}5zBFoEjJ7f|nU}3{b3gwUFM;|$ zX-%>Af}VSCekbcjl;*ebtr3blb`xYoWDAW?PwT56kadexHh|1BBSwx0YVqcY-OWaQG3!Qbj;RT!qmaUfFuQ+tz;U@xuW&`K>6@T?JK_)6 zX09TSA-1h80@Y zpXoT@3O(hTdZZ3$}{o@H9((;?lh@Jx{ zYY#aH2yxDybrhFNRM<`TtRe`i?k9`4R8)Q|duEa<5wJDqSxaAcmR|DT*D`U8A@u)T z%jD3{^bb;##7w=Xnvi1W-8TOJ;Je$u+BaQ#zdun+9!E6{Rbq$uN0q6b$XpRWL2|#! z0^B2LOeu7y#k-UX@AhP2-#BXRZG4 ziin1`Yy)z^K-o)Asj#wVF?)7pc-!>?oaSs1ML!$LhEdh4?limy;SHUhSl|E2h*v20 zOK$Q3%(zgQ_+%1vuIWa>13H@Up&Bq69kTb{o_g57anzyi@!vOscy2GvofB{8ra{d5 zO)pl}XJp?*$fxQ^F-+s}TjGL7GOr)T2rYf6JS$`R!R=lWP4>Bh7sUwGQyY`*eNcpwa% zb|Y-$VX_=i5lysEd9dE?u|Ie2l7)oC+!H$RJ?f;S{&jc1p!2yoEj%h@hHt-4^YLLj(fyB0;s(){AdgRS2j8bI zP$o6jNVwQR<4L{10U&K@W~`K#2jY~tXX49Ah3L6)7u7o(O}FWSbXc)$KHQDn(?qU@ z)!^{f(IW-S;oI+BJKku{sMlPHupebFjG;qrom)cI*1}4<4?zh&*=LXD(pbHq@$Jje zhmf46BA||sjfaC^m@XbXz3v8gp^-7Be7CBgL{L6QmX2ifn5!MJl#^udnwDcnv z6|bzaXfrnsn_=3K7IYNvx59bM92A8bu=D({ikhT={jb%ld|XK~H~zbolpfS3w+T`t zMZqKZ$&UQzQOY4kSe^^xb|(8VCS3ysl)|aN-zlN3Y_uFILOpP=cLd zF5xjo;tu!T7%t$BXonVieBJ+gEhA^BG;Y%O+tA>FohPy>M68x8ZN_(g+arn_zSQk*Z{N@V`y1{Ai0+bAWNu(#Fi+XoARHRtNHb?r&UB$82>CIj51_uX4s+6a6)Y^|$ z5X`R(>QBVDpIW0vAEUILk<*p5tiX8omQ;`4-d7Y64xb!MQF~h)mBf47P{_4Is zNC@Xxzz|j@vYXx%gK~C2&Zutlqf$n*lonNKI)VeIkR6#>y`V@xQ6Z2t zM~aUnQ?F=Hy1PSsPwE?r6baD(C@2H8JOXZbgWFekGYbsoOatdO-uhF!TKa+KC=N7h z$QB(O<0B0OF$5E_aEI&-8?<>r3l=OkBRRi5QE+5A{gmjpu5$w^01W7#+?NET@YO91 zThGTk-Jc^+QDPgq)ZS4W`2cHU25lJcp#v6!ezv;NvR=x;g=wBDr*Fcp|s(S-;N7 zv@2i7Cb7u^<3^nA=O0$cGm+!7r?j$Y$dQt<2E3TU2t_VwSo5}OVQbtdV^!ANntHhL z`On;!TM!E7hpFEzbn^JSb;y#!c`ek=u4xZza?2vg5BDko*8fK{XRZO`t+vVR7TXQF zR~()vcKoeMW!>pey~@}K2CF19!GD(*T}j>WZl6nSct524Czz(02l|^Rejm&a<`({u z#34r}3}w#XsUas8Tp^JiQ0=7mTq*&XF`ZCfn^l50XTsv*g96H6tTEf5`?sJ3H6-wy zKB4P3^$Ya5OS|uI)3u)&U3{a(;rR3n6pKMx=h@)6p0Lo$@V6Sq zs&7a#*8ny)N>9dl4TQ{uX6CCUxmYi&YFvT)xGYvQyj9}(XetC~zl2Nv(1*bfy<$_| zi!mNk-Z_W#SC5)gGpM9K#*o(U;iV(pP*!?;tGms$J1o^V%kx+GO0Xvxbs;j{g ziF7|+7yi9b(lB%RGtF8^Uj_IOo-@4Z$NHd|6%jQ0x#+#r8(D8>ZyY9{JfXNG2JTwT%&N;T*4FeM;JK_Zgv2ImH%lRNn6I&_PzZ9k6T!aZNee4FQ4@@*!Iqu=6 z5I8OizkGBU*73YDfzaxe*na`en%Dmd&hXk>l>^Zp38~^OW>V`~zpB#yMK^>4{lapl zveW~pe(Qblo)f-`zGQPz+j92t!Qot$kgr4UQFv99xb++UrpcmdJ%M~^F^cSjO%-$g zwD-H+$3*Y%#T-HlpQHCIKisrIn0{#CPxAiu?UVPnlQZvc`}N{az31nj@%x>>lu41cJN_s)P}m74P@Vm&|RJ!2uHZ&Ck3xF#Br?&5gev*XviXP`UQ6#XcQN;KxhaXCG9IG!3|STjtV7_ z7FGA^Ju+uemc7I~pg6JLnU`0N1wxzph)ALTwS3^fwd<7BvAl-F37@o5ZoVXemxIgG zAU_0c8thB_Zn_cVH3va#*?5LzPLZ*vSEQ{ZA8+q=GY0x=99ifiBrWEPpOkX>#VWlr*|G9_$-(abbDf^YTSM(4hp#MtnFX;~Tk+_mnE_ z{;@#nhd=t0RNhnBcY!R(S580QRV{DD|E)Ba8Um{IyO`$xFbAZ=fl73AK+peGF5-8U)}ftBp^4x~=ov z67#@gXI2M3cQFnd^r&XE$bm{foxCGB(BA~y=Y3MlpG11Bz)dTaDOtgi_dTEF2f6pP zNi{;6=>VBlowNq`cDas`&$8o(N4xVo%mkMz{q@S%YpK!}TxohK_JRyz@7e!lk>72h z;Cr_HAm8B{kiEx%(*Uj1q>JSpv9~64gh18=D(0V@)Y1_-SgUVuaAOFy7QxC$2zFWF zd=2MT@1R!xWpwo!2J=SqOCmF!(pCKwA7pre&`JZ#2`@YY;{!Uam&eh-8>P0!Lh((2cY-C%po{e0*blZ|I&OSngRxE(*N6N#E1{CXkD3~28ShV|_UkEUll z!S*v<5pVwh+*^uRNv1PdGd}5wI@PtzV-V|c1&E>j<=jg_*x#sv9vjRPi^J$b=u7@N zp_cGff{DO0CjAZW_x3Uho&OzNN+$&waY^*q8@5k>(y~CLRzCNxlps(+#>i^i+_v&BkSFbAk-}S4q3PK;AQR;gU=Uc8jtw}#~|Iegs++dLX zn7pv(&jHsYU7|VyXjH3y0zIc7pc2LHapP;6b#P=hQmj@9NXuS+@bEoVynWW)VE5n- z)fTY$!>UhTR`nCkP}NTJd@O`LUkYq?P33GV3PVZPZ-<3)~{N` znB0$8^3+6%fO+|KEfn7JXUs4cXibeh>`uzyWbp6v;8+HLz8D|9e_`_)pZ2UxxRCej z_p|qOW%!QAbsGK;=p<~=W98Cg4x5y(MO^?NKVd4w}+;0BlA7rVPCwKHK@ z%eknVl1j|Z1Cxl>Q&r(z?ZO}2PYzug=EE|OBs(G~R_Wk>;eh`eZpnW-;IXSC{*^oW z%xDG0J%nc$fM!95-dl!ZbNm;Ztn6_tXvudvbcq_M&dzHy;cw#AYpv-*CW6hHn@^ZM zr7|A6I%FtUcnl<6;t9KtLscSf<1Vj^+!ufWxuJ&Z@lq=zJzLpu(Y;Hg&o8C)CS*V|t#{BBnO|C{#( zA_X6(S($lvcDzk?N%DU7kw-RIJ;+yT+ZQHBmG0rkw{fwlUlzxT=0Y7;jjsH>9BP$T z$k1%LD-}sJxmBY#bv5xt;^#AzGFDHR?Mx;*csLM94$~gBdh;OGdHu;}pgfl1QyB?D z%~_=u25qNgE!`L6Q@LbUjCSi`d|GlT!Vh$pD#3P*nFZ7@x4Pq=i*cm5t%!SD)nyA| zuZ9-Q%_(e%_Tm<7e}}9&Cfk#Rmpv(nEC29iDL?>{KfwO(Eri}v9^#qEGQW0+Zp(_~ zQSd5F(&Za!{7=yR_JAMBq#dIb3_=@RY;Vvz?x4N>TvM~`M^zR&aiL`Z{6_zB1FqwU zMYlP6Yu(-dz0M{VNpqkVA?FF3HW{>GOffusf+xd!ssm@A5<55oH6I5G+iQMHdnlnv zaPjLB*mqr6S!jnz`3OEBVXWLI%yw z>Yv%anA?A^t9ymwJe49NBvYlV_DUllj&#;NUi)@95NeRuT6M4=-|J$$4WRI7fz$-& zW8+-@yMpKYrvDxipn&2~W)1AQN;uZh7@5*FAPpO94-apKdM{7YG`-PeX$pz-LYDL< za|*`5&+Y)x4}Kp1%SYrJ{Hs%>#T%yAjP2mgv^Uq22N7#lJl+dDN?Sxm*p7<}w*RrN zwQ}g0Gb=_sE6KrU&8XS}j??wCF7QKnm&cK*oSoZ`&R?_^|0jx#efBl6!{*P9qO7be z>y(DCZqks^oiFA={)aF&^Y`R3F`mMMRIZC((rBL&|E>Eo=+5=dtB_8r`TlkMXgd%w zIH#<&YBaX{FWmO_JbJWZQf|I;JA}tOkxcG0Mz3@fG%2H8MY>NNscOmg$EsnJl+O{y zG~|)4^W6IWzlmCvf`kx^!ow=uIX>Tgyt!7-xGA1o&nk=v)7)K+FilVO-+d3!-(Rds zy>w4a$tHzNEXPgwfA)P(y}PZ(?Fj^!YMf$NykfawGavt1icIxskd1;i+U%ZuM+azjElVK= z1jJHZji2`%YOj(JV_sqcmGw#1dc@r{2aCNZ3e&iIlV)wg67tnshC|dCx6m@~kP8H< z!#p)BpgI0(5I{UA=b!`bd3xRxITc(GQ@AK?{Cv@8RFOpVeEpRlKj;U;ia%%7z-G69 z03a(k4f;wAZZ{OC6zYqz_9`boS5wOm3WR_;y8As$I#o9v^$|z|1J$81G%pvM4{Bn? zsIQ`~8a*70J&tw`J;^#vE=nG9wOKzRcswJz*>VVg`}k!h!TuH$HX7#NpmWz?Vz@wf%YfGRKyklCkFypss~JUSndSW28CjE)Fu&6`Q(99_`TgR zw2gV&&c@!||VUoAXvhdc3fLke4+>#YoP&#yj|RN&)x90xO{bMw-V0(+Nj!kE2k@$9d+Tdc;NUGn z?r1_#BC_ffJa}Dp)5rO`gH60`kAY0!*&F5S2Hve^qkXx=C4PdVp1mV+cNJp6pI(c&_I zrhcc<%}}5Ttn)hBQ}f>qVMiVDCRca5mjJ`aaJuV(Q>uYdckZ;ajXhL%9}o}h;Dl1w zE2EK^oCy8&#apZ;f09}PFaC-Vg^UMcRNf4hOL@pd2=`E60bY6`KD>X{yll>qA%Em?OX*NB9`G!s_-!$)$m zR~Lpx%vehT2+PyKSRI%VqyvD{J#Bp)Bu(_h|B`}A@H^FHbtB8E(Z+*6z@S~DZlw0x zRC#<*m@b5$;iK)ifNXC}{~9u8OxZ9CLfn{IXs? zz8}1$t0vT1UcL4$rc{U^bA6haFRwx!+^Pn5g{>SWKjxta*(vIE7@~?ji)q z!#VkHWsjPuF~(kN=$4k2@c*g;z->WVW=3A@Gbmd5H8X!G?XG!u<}|RI zbRZ80fCEEP7e7~*_aVz=ca6-+{d`b6EvDNHt^`HA3PEnY16Zi`Z5&EwK_MHthwJ?k z$62Un;_|W9Lj9CKMvl*Q=9b6Oj66qB zBo|IIO5g0d$uAe%=FDbCUtZ1a(AgOy^|>}*U&JJChhxs|{8mJ<9!msST(~GFOKLr- z+}PGIvc5E*{djxTufF>Q&|5+RPumUeW-H23=m|`1Wm+~SF?b>cfxKwfF`hdiYbrVc z7#^dOh!g1&E6EBFJM0ou5O$~${dm7W6JY2tX(_&UuQ+kl#zy9Gf&jTnKGycyKG-g4 zsLkgNP`5k1v#@YQhK2mrypF#3^ry0;j1g@m-=XaTzY95OPU*Cmq$bGw^`q|#@jjA> z>#Z^cl-vXYK+T!`)FJz&lQM&$5eU}`aN?4uWX;{*S?>1-cajSX6ZZ_mPF~KxZWQHs zwo)RYE1J=`*S-ir|`Kb?_Ms`7IMyu4*N%zVdHb8>+}nYHwspF1K>!j^8wwvB#K zZpBnX;TTx{WB>=lM~=wj0Y3sV;YBu_m16zwZ1Ln+w>81`g|BhZ0evV631Y7HP~SSF z!I?&IGl*$U)OlGYw3W-?QNd+sX$7S@+AV}a{;ic{Szt3(!1T%<4_bB$mQP-m0iEX!a3i*h0h)o0t5!hHivICl<$dWTkq>v5tWe6&p}RlUi{f@m7#Z{rtY(v zMZWK&qT5f3k~M9!`(kraOtiA%j(=lQf2%_cSR5c}puA7uUhur_#*B9_258LnWzSQpSGB2t3{>C9$9g z?9n~bss``{Zvb0}9$kR11CJofgR#l2h6;OzNsZ|8Ij5S>K#k;4W4p+Ao&a+yT_ab@Y=p?=p3|Dq!t06X|05&GoEHqvc#h`LJ_>gpV;f zv-z+3>A#i7tCG@(k5f8-IkpoFBi`!?Ov{cVE~Ux<*ifK&dh}o;I;F1}j9+4^Cl@TA z!C3^nGdqRRVbyg>Bb}}M9PHIFXCVHuwxK{|_~S;>CD1XN*-LuU2sKTPu!M)UVHb;E z?=vv=vat@_JYl|!8Rim1e}r^>6ia)KNclz7ehdB=5~R0)3&!%ez6qX1>#KAh2d)kD zHMvH61P$EY$d_(6iFiw$5`nb@fFZWw>Sl(K>zUr1@_OXh7zZi=NzH|MGRd^K)rE&jLUbJOO;JC=@m>WuZ3>w?^@z^h{k z9y!jukmD=Y;+(JE$%um4jzkn^;h&eU9dFlDZ`~b~G_4UfkN?#CL?VrkuJaCN52!8& zuFg0G4OBgfrXP19_{YvRd+!(5bdafqq&vv*%`WKiUY9+EeMIrnR@LJAu7f#Z$H@}E zowV#hUR#&nx833L2e82WotKyQR})A5rF@4|?RS1mI;3vS_VMgD0e?4mww5ul?E9XCHdT(<1Zp|m|pSLnAL zkOv)j)dWW)cMTVXz3rE;n#vQV)G$WBS7Z%&3~FBP4<;dB^=APW+c)@%wI{Ip@L|Le zFPxfR_H8lF80f(EU4heoG`mE|j?h`2>^x@QJo_6=fMH9_I*k!=y(k1Pwq$$VOE?=g zmf|40QxV=yEL9TQZr9gSeVLONG3$E+EX<}3%~x;kacfGHmk@UIUBu~#w>s3l6a=q@n6WPlosnZRxQMrYeO(-*z1NQlZ#O=T09z&T;j?`UO7p3;-*+k5l#(hITd4u2J6tK9Y7OqxB~{?a||a9{$s& zPnD>%?7-SuFg=_RrQt;Zn;zj;&EWAkTpBRHmj+F(k<@xx{Qua5+ME?sS7eS;GC zjuCJydhjXT3%Dnh6DO!lbC1T6_BK@_(1l}oyG!P&=)s~F#h|>`+&ulrMelRjXBJjo z(3yy>0N3hz*9f9z>fZS8HGrYCUAEu?XUuxvkazkjZrmXDV38jbDT`F8yZ7)Pi-ci4 zBW1Pu3v7#o!UoPS+4_wIVh0vGrVLn+VUvyujo>-2Ma7ycj+Tn%$I>QWr7a#$Z@>CX z^5P*$0aB^z=^M}=oaENP^#X+p6X1^0he1tMR=b|4Myy;4=An{Q9?3dSWquNYV0dTA zg`M{>2$GlP!98D$x<%4t;8{nsmKPp*Al={zD+{wLpfy$q4n21)?Ib|Q$@A3zBxE!Y zj<{h(MDTUFR0aTz>}{fsmNK1tDb&N|tp@Fw&Uf!+$y`orzf-j@KU&C&$3!tMFhi-+ zsh5DFrmyD#Wel)@{m4PrPQ+2DL_%`?fPqgN{uFTHd!$wB(DHHHsg!O=IIy}Rm(%8H1&^wpX#&@Jf_{Ps6@PvNmQ zj@SNB>xeZV1Fpw!Ppj730$3H4cB9qgutbB7wh@)#W9Dw*Y#w)#)Y5sESO_0&E{5yB zu(EKZA^c+E`VAwwtO8k~Hwev0-1EC0Q>P{n#bDq}k-H^#xW@q(tvjw?I0yu+xz6d( zva{02G0D-6FdcYZk3DIE}uW9j1#vNUZrseC{rZ;z$J`0U&#&}HI#GjJyuQ*9#@B8ki*~; zZ}=3hw3F5x=+~n93aM`AcnT+Uz3IK3mT>T`fxl~a0TFnyKADNDuTIEb-55wM6q#+>gIiz^IIog@RD6PoNne50!N;h)) zC0EvwogGtz<7yIcM6N_9G!32{rE33mS}}K3_Kyr<_cUVpopnBVLKhs zlc`OLv{S2>01G5z7ecVKEhpa!bS2+kaCaz>huWQ~q*L75F|EO0G0+E*a2X6~90tm8 zT!!gp1AY|vp~jk-;y;GxSLBhwRH@6>X2T(C*x=j3mD>woFhZCiQmX$>yJO9onMu`V zz3J{m|I4Nw?TIC1DPb z|L`=LkS%2Kn+Ov8mGA1Uyvc*y4kIYJPxgZgeXTdNoNAZe;8&A9nrTE#B_fk;P)D7w zh<4t|83L-Of0$Ue%WJU z!6ng*-|ZWP_%zg6LbSSfmS7Nr^p{($c{ipI!oLg&Qk$r&6S4*nGYkxI%fHu*AZ}MP zdb|Ana58-#F1dTM-0hBYK|$wo*<%Kq;~3DysxG@%b-DAnKontkW3yd^Fx*4yj`c9H z^sQJWVdBP!%kh2&IEke9T|%SI6HTnA6pC_QzfyN3&ymnd8zW$}qW8FJvkU+D=9(2$ z))m)f~9;s${nU@<;5W2ls>XD~ou5Av;ef4gYr{CN$R71*4 z*G?bGa34^SyuLkEfNnEwc<}>M-R>}4YtMc0*F!1dkrJcwRrB;TGZqOS(hc7R@M8sC z=DwAdz(u1|Q13pya@s^j^kAe%vzQ@P?ntWhx54(l+MxGan`xJIP`S?e#cpi3t$v6jDK6WH^XmQHF;$ox7|u#lPdm&(G_1$WB3eKzMkdg;kSWu(~wikR9xeDTF5=hP=ucd^`pxw))^?y%^Z%xZsP!WX zH%7!+n}b)4CR_UzQ7gXSU98+{GR7MQ_FfEnly6bPn;SH{?%?2sE>B)_S0tBQIto_z z@Y~6Zuk~&3nUhs_l3ke!qj48>=s{eSuhsYLQ82|JcBE@(XOg%Dt{JD(DfxbTgAI*@ z@f+{=6-p`0yI4PRnq@BB>WPQRKJI56YkKB5YTQ}w=F#dn)PK(D%XojNOZdsXe#smo z>H#*}BlUhvtpxYxAIGBI^ZU2quE`q{dgh3?nl;l2O8#?^Pvxae)^VvyUV}-=4*hT9 z6^T~_mR|IG^E>{Mkzvps^H36jc6 zp0{R|5vGRPeFwb!ky;^CA3K+ksn>fsxRn1B8VsPKs&d|7Yu6gaH0EA|)dAG4)n2sE z-rKn-I;E{?Et8eG7ntoTq|-cpttpbZaE};Ys1Us`eGw>+{~=uKJ}B@*7`W^Z-ehIw#0OfZY`C1Ko=a5O9K0=0`Yh4 z-730HFci^liJvs7Xb<&%fGas%N9@{MGz%qM0AzAQu%|uyahgNL@IK?d!AUM}SSgPX zVWhloc_g=`?FYD^KpFcGgu#v6s4;X<#;m6c&>Bywz-t#ZatU( zn8#)DPl4P4Y-t818X4v!g2@XKn@R#y$-w2Ckq1)c?pSAS4eMSJvNFjbd`n z*XUq}%+dN>`anb~g~4V09e6JK?Zz66=~375QX!Dqo)P3u<=j_T<9#fCt46KDqr<7# zbB7n{f!-<^GH}|vqTqs7h+4VtQjxzImCn=D&$20z{ydPTq71t`)_8BxAmTGqU|Aud zxox!P)Y9bga1z|S6y@A)Jg6i-|V8KC$b{D{(X-ui@?^wZacjEM-d5yz**&5{iaGNtrjNA$O;jTcf z3&l7&^wa{Hsk@H1WaZIA!>+h;5WLqDcBI~}PiZKDhD3K7)&7AQePGZ9Ui_x&3H!n^ zYDhKzmK{C1lTm2&kK&R)LH*m3zYJYyoqP|T@z2Mk-9B4MrqibX^LB2m?Z9E!4mhXN z167}X^P;$-OveU}(FaqUKi~wXng{+59t`{-08{>xQ~^GGIQ{?Xoj)lb;G3!jz*;JD zcBG{ImE3z{cagY8CMlpGSyx}c5>vv@{Sxz92 z{pS8{jfnDy4W*5Uq_|`C36-p#FpXdM;r_3VTDwnA8)P(+=ifWj8}C!3=wRx9(h)~` zBGRN3R(7SGOrBI+_i~Z~VOVJg_#b)f^N=U^X008zTlZW1FQn|x>^Z-nT%82`IdHwd z9lPn3n4F;Gvw81n2qRhpU?w(iYs_xP*Rv=9L0XL@A%-D*KW&`toW33lR9tzS@bdI? zECCH95cWw*`!1>2x>jH0F18N#R>xs~jAy@)-~g90*%=J<-)-3^`IG(gO?RiNFtDDB zqir=x`viIg-%i7}$gM4EftT&ng?mUYujQO|6#SvmU5@Z^W4qbk)LJ$DwNrM53X(PA zuhN`H?6{n-HHc2j7u+p!3)Iqx=O!80m)s#DSWR#<4rQ6;FIqdteQI-Pq{NaW{NWy~ z-ZivZ{V9Q?rG|#J5K?Dn={7aBciuez{&0*tOcS)CwqKBA#>3V7i$_891rKr zh{d1$U)O~+Syj_x)t2muypQ_%Z0YSWBVxU`tec{5i|ZWgQ-n&&>OVM<^uwXnna>z` zf8pT+y<;QcX;%JBC|`fRGXB?C9f$Faz5Nv=N6G3)`7tbm-?{-ybmfWcohB=t<9qWu z_`Qw&X@8s#?Cy$Rt;frjtW-ixR(g$=(jF<*aHkMbIc){6d<$E-XC^**3q53WLmTNH56FeI}jUw}CjZ~WT84%l%gvkJ}AXb9JY>$Fn?@YmhK;sZS4;3wCa2Z$`wG?z);rB})i= zNI5e27)gFCJ-N62dgp^=WqwyIM0k{Fqy4L)zkiJQff%m?&1)$`^0cHHLmVrM=~F#_gU{_`RFUlA z_jTd!#F5gfM{`hP$&1Z{{fqO29XV<5i?m*iw zo7k!AtSKryZoruc^(@9gFSAoc`5Vr86JtdD_UCuTcgA*7$>gw9P>w%3xL{CZc5Lo3 z(kXn$#OmQXSiUVYp>UKO>dP97Rld%3t-wabp|iBZ@8mu{93AS|i%H$~@1UY0a{dHY zKX!BMwui6s_VBsqCwNZBr^vZ~L&bs@KHh`c1y?4NV&5(rL9q=Bm?)DV?IIDyP&`9a zp@6EJ)P`2?QhB0qVqawrO(=n&wEgCocpSYKmmA-sj5NbCW zQc~5I153gZQp@Xd*61>1Vyt(r-q@Kmz&AtkTRxnjZVNI33BGi{Bm}K-rx*hs=PkCFGm%GMOMoLW`6y>xfOwkcNGe76_b=3!QS z0ZF_Sxis56Js#&w@LyNTM30HUeO6avmRf*{vBygS@MknZ^=s)((b1)T>-x?W z*$S5qRvv#N9&J-Z@ty6Z*|0)^(Fz>`Z2c#gidKxm6Evf^@?q^l@HNxaAyZX3f^W|K zJ>_=`8u=Mdm*+G`!>SQ(xw~r%M(aNjMC~)v5J!m**b@7V)DAduYuC1UsS-4?7FXZD zQ>j(d?&q|<-W~;u(;P9pvh!-TT+Z}Hp_zz@#baULtE#k~`q>;EPWs6lbXE~7{dADE zT5G7?jj<#;DypC0pWqc7$w)f-Wncba#mBG4BkLsH#yAnb&nZ~BUP7)`w-0zRTo=C1 zj}B%oa~XLCRaZArLK_<^w2yvuin_-!TB;9$4P)7ns-zKkYCIu?-YKlJQlCF5lV3&$ zO80a+7vAlcv{46_FD~-US?X=!W0=c8a_v9Qt`?%AdJPHz(L{-&*Vr+eUwTby_dfO`aFOr~(s3GrOH_a$UP9Mj#Ug@5ytxeP%~}RT-SHh_SYTihFO9;<013l@Q4utlVUQt;_PV1kniEVKA&2k5$c`>XL5vXXJny z3n}bJI{~CS7|whNJlY_+`tbW*CvOE$zOg-`5q1z9p*H1UP)aJ{$6sx&A&yLNT&WDs z`F20$o9m;HPb;Trv$-4OabBPFcbhlu-M1FGEBca4R0mF-u|p0!wCjqWt4bpEl(cC_GGd3~Gk61O z;5s|`z;yXGcrw4}YA8JZW&YVuoz1s3P%>*n#ant8fv&d|Y-2fn%yv@iX#6j@= zYcBd9+k^F_S7*(%veZ?cc}v=tsYqrT=X0e;%6Q-xvS^K)%1uV^<_&2>ahFrGH=Gy2 z^gazSqbyWZx1cIeYSjIW<$`!p6MhcF%1A}^{D2EV&sv5${X?3jx1M-RtJ|`<_>z$2 zrV9REAA5QBtD~k1Efn&~$W?3=ST+6zI+F6DLMy%RwYvn`5!GrCG~29EU((M=u)){Opat zY!&5mdYGi9#6q1rUaAYia;!&ybT_KMplXEBM*x*_5q-qy8PrY}RuWGs=a zLFOEO5?^O;zN0+4+RHQDu|_f?+exFfz1z6@-o#K-QOSr?)cEU=jYkleqQmBVXFIrW z2H^*7CkOd87!!~jBXN%`C2&^aHrWh2DiV`3h z-$XajP*J_IAaW8psi>&xY-eO=fbFJw-AB=xzzeFpXkgwKxRlX{PyMPyZ z5tQA<&drmCT3X&IJ4O>;-a}{)^2Y9K`B(R)>46mk>je%jVP=QIMblJMr8@1T zPHR7<%1=G6n+oy3(64zv0FRicR_-OjwxX9;}>SfZbI<+B%^PZ+KTOd z_reTJ(7DEqfx}e+W6hYtoseJghte3>0uB8lffyPtgo`ZZ7@BK1kYB%vj=1iC z-N16G)vl>$Pq>9Ay1RSXlkxNDjkX7{go`M+u3&*_jI zZXZY`tQ3vC7^Ux(z4!j>$q$Ae9VlT0&p=(OF4jJ!>1TnKj7GBLL<3nacLVcBO3?m;?1*lvIgDR!FC*SUqC3&@Veh^LJ2<*%S)Gr!9AeUn%DwsxmV z-Mjtv3{;mnp$r*Ns3-OT6l%JK0s-WkMXQ$b2>Q_ApgYJ{HSLv@TB8EihXIVcwv zosr64X(>xq{ICE@L2_#=EayGce`jTD%Y7OD@ELKS1O|GQ+x||s5=1}{D8d$rT+dCP!IYe+?>XOBR*2k z7iHf(wOSNWN)Ite;!2awze+r63x< zXllOMGfn+bW4d8JR=TyLkUcVw(Whpd;E{C2#=j%0Tf_}n1vR!Em)~0v8eKAB9ANbM zNn8vcoAh}&eU8zS8JNU@^Ry|h+nnZCKWO3I^ZWcv;}Dy*OPzHJN7e5iuk)WShtb7n zG@pHTew#Q@rXDro=y5k^F)qg!GDap(UpFwsj%2`Z-j`As{9Mnd7hEx!^prt=mJb;E zz%^)n30o3}_|Kn6{?hHr?<8r~{p+=6Sm;AE`}f~7uRrvEyjRFUcMn28yv9EmOgr$| z3)B8?i$CFke@HU|6=LF|&auNkH0-DJWTiwY0rfhh@+6?{Kr}>7r!pqX(BPEfX6E7k zT(ubLw~He{LE~w?xSTm`CRq@f!1LTW8L*2+tIB8MGzs>n2N--qeD)osqp_MUe^-7X zQe<_d_kxV{B-91%01k9F6R%2Cy!j z1B(t3`Xk8`ZDWzxdZe+*?(vJncxj{C;rx~e62lI7JaPuzLezXE!GZbQk4l@61HI( zg^P?CJ4R8v)DVU_EW+=+oqY$v*Et+Cfb+DwfnI9-mH~Nhcb~8Wvtk{SH@W zjpl3ZFs#f+(bxE1)_FV&9G(=R3V-Hq1BeCPcPidU&%0MZB!RUbkWF?DVc+ws z*2mJ|o@#5r5z@lQBt+OY_u|z-Y(W%*QV3Jr6<#drMDkok3+9{VIB8JbFof=TnFpWU z$Y1hlPQG5t2DTyj6XmFPVsHaQe`4u>Z5sgJ{QpoG|Amq9p93XFEiSjWEt9fa6i94P z(Vg1i#ixF#Fk}c|#boePoFLf~Y1wF}s6X?7BXI!?w|hBC})#j`msXszWEbF zi|JM3Nbej1W^S78+l?LW?nAB;+z=Z~e|DdF1~D+zZ|gLaNc<5anQCi3`n@FeOJkyy z0KAVik1?Tm)f(bB5wgjDy+Uez12wpB!s^2aESsXwWGtcHgQ*^U1Ic~K*~NWn5v}@7 z-C8ZI5fv3drag<9{5gg46{|lXFgYjEVz_D9ep8+!X2&G{Uk6xRORQY_RA4PCRBja` zbWeS@USJL}CEiXN`7JB5!+rEny}jxI={kfJe6knmSmFfLh?W+Atbkj%dQCf%j3gmK zmF#8y-O-?bAKM0+H?C1MIwlnt5Lwfow)MKA7{eYU!GNq!<3Y(03}oD)i&u)?hedWm z7kvxqYJu%IPnleE_f7DZ0JgYsVB2|w$=zgX1ykO6#X{K_PuFb$(ag-u z@pc7(2T{Qkw)@Y`2j2PrF;)LD5K6mbaM_d~MLK~uWnJHK5TAbMF#X|o+MZ*~ua>LB z%_#di$7X%4ecw{_=dSQA&NuIX?WmwUfMjdrlJDYiI)H$*vTcdu5{h=mv|4rHxHK zlTZYhl=M_YSHu4^&1P+Ppc;d0I{|QlyDm|9ACY+D&R*8djN6@gkkvs8ev@U$637ET z&I)++6gwXHK%4eF`O$~mkUWj*?&tTNZLzr)rM~a<%0XzZJX8V67hp$m$`PNO_ueR$ zKXVl3C30uWrRR-G+^@o)Ne`u4+s368{batR>}?ze6b|QSEEEJGHu8{HYBt7p$Qwuz zM&LQ@h}?S*mXGkZ-f*wn?L4U+lN)JQ+xolSuC$qRKbDe!aX&NR>d=6Hht z?lcdPiO88Boi`OG9TM-_tQmhLN&Nn@H^SC^)B7`Fm>k+m8b*;1HiV%S{9?iACwL2u z+!-Y_6rMUmAk2-SVodVgoD4Db-*Y32rXly1os@u-6{9>F!`Acw z8mKNh7;;%+mM^k3wT30ZM=Bxj;~FcR#VOa=rwzPU3$SPc0wVk{l8z8y?KqD-cllf; zlDt+Jay|~Zzeh0jUDGCmt-*xa9sJLUaq>F6Z(>4GmJx8Ubg$^nYUhkq49-Tq7}h~Q zO-fveJ(poZ*r-}6a#-UFlJJqdL)xiOd}>I%c1Q(BC>?1!7l8H$;~EMlYnWbmYcDn| zAyW5uu^}2Mbo*>K{dY~t#CS_6T}vC7$Z3?u2Ei}eCt_;7&d)pcs|O#RUK@HKdL3f|7|@aTY^>VaHI=#EbzB2Xl-;7otQAlQrP~a(z7r_lJ57h?9w2J z!=;*Fi#V-MbiIR#PZyLO+u>+5SbLt1qw(<`Sf>=*&}%B4R(TJW*eGzQ~?I3!{%+P5zu76n01z~)= zRleEu!CFDALq;-Ieg(UO?pvFT5;WDSEC)vZ#B(R1g^OFZ{kxq7+NIvanftBp6Sb&ZCKy!gMo=gY7kumLC> z&Omp0>trb>a9zHft};j9DdfdP3I5i_on759tgw$Qj>r8-=Y%#Q1!rz|DCHs+CR@GFnyH^ S)Bt>i>i%8LJ22(v0sjYv0pCgh literal 0 HcmV?d00001 From b0f382fea83d7f7cafb9fe3b3a26b62387f2d376 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 15:25:40 -0400 Subject: [PATCH 16/37] Add logging to MemoryExtentStore --- src/common/persistence/MemoryExtentStore.ts | 58 +++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts index 665381fb5..803d1fb51 100644 --- a/src/common/persistence/MemoryExtentStore.ts +++ b/src/common/persistence/MemoryExtentStore.ts @@ -15,6 +15,7 @@ export default class MemoryExtentStore implements IExtentStore { private readonly metadataStore: IExtentMetadataStore; private readonly logger: ILogger; private readonly chunks: Map = new Map(); + private totalSize: number = 0; private initialized: boolean = false; private closed: boolean = true; @@ -76,7 +77,6 @@ export default class MemoryExtentStore implements IExtentStore { } } } - const extentChunk: IMemoryExtentChunk = { count, offset: 0, @@ -84,7 +84,18 @@ export default class MemoryExtentStore implements IExtentStore { chunks } + this.logger.info( + `MemoryExtentStore:appendExtent() Add chunks to in-memory map. id:${extentChunk.id} count:${count} chunks.length:${chunks.length}`, + contextId + ); + this.chunks.set(extentChunk.id, extentChunk); + this.totalSize += count + + this.logger.debug( + `MemoryExtentStore:appendExtent() Added chunks to in-memory map. id:${extentChunk.id} `, + contextId + ); const extent: IExtentModel = { id: extentChunk.id, @@ -96,6 +107,11 @@ export default class MemoryExtentStore implements IExtentStore { await this.metadataStore.updateExtent(extent); + this.logger.debug( + `MemoryExtentStore:appendExtent() Added new extent to metadata store. id:${extentChunk.id}`, + contextId + ); + return extentChunk } @@ -109,19 +125,37 @@ export default class MemoryExtentStore implements IExtentStore { return new ZeroBytesStream(subRangeCount); } + this.logger.info( + `MemoryExtentStore:readExtent() Fetch chunks from in-memory map. id:${extentChunk.id}`, + contextId + ); + const match = this.chunks.get(extentChunk.id); if (!match) { throw new Error(`Extend ${extentChunk.id} does not exist.`); } + this.logger.debug( + `MemoryExtentStore:readExtent() Fetched chunks from in-memory map. id:${match.id} count:${match.count} chunks.length:${match.chunks.length} totalSize:${this.totalSize}`, + contextId + ); + const buffer = new Readable() let skip = extentChunk.offset; let take = extentChunk.count; + let skippedChunks = 0; + let partialChunks = 0; + let readChunks = 0; for (const chunk of match.chunks) { + if (take === 0) { + break + } + if (skip > 0) { if (chunk.length <= skip) { // this chunk is entirely skipped skip -= chunk.length + skippedChunks++ } else { // part of the chunk is included const end = skip + Math.min(take, chunk.length - skip) @@ -129,6 +163,7 @@ export default class MemoryExtentStore implements IExtentStore { buffer.push(chunk.slice(skip, end)) skip = 0 take -= slice.length + partialChunks++ } } else { if (chunk.length > take) { @@ -136,20 +171,27 @@ export default class MemoryExtentStore implements IExtentStore { const slice = chunk.slice(0, take); buffer.push(slice) take -= slice.length + partialChunks++ } else { // all of the chunk is included buffer.push(chunk) take -= chunk.length + readChunks++ } } } buffer.push(null) + this.logger.debug( + `MemoryExtentStore:readExtent() Pushed in-memory chunks to Readable stream. id:${match.id} chunks:${readChunks} skipped:${skippedChunks} partial:${partialChunks}`, + contextId + ); + return buffer; } async readExtents(extentChunkArray: IExtentChunk[], offset: number, count: number, contextId?: string | undefined): Promise { - this.logger.verbose( + this.logger.info( `MemoryExtentStore:readExtents() Start read from multi extents...`, contextId ); @@ -212,8 +254,18 @@ export default class MemoryExtentStore implements IExtentStore { async deleteExtents(extents: Iterable): Promise { let count = 0; for (const id of extents) { - this.chunks.delete(id) + this.logger.info( + `MemoryExtentStore:deleteExtents() Delete extent:${id}` + ); + const extent = this.chunks.get(id) + if (extent) { + this.chunks.delete(id) + this.totalSize -= extent.count + } await this.metadataStore.deleteExtent(id); + this.logger.debug( + `MemoryExtentStore:deleteExtents() Deleted extent:${id} totalSize:${this.totalSize}` + ); count++; } return count; From 481c5d106de072a073dab7b791041b8aa722e70e Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 15:25:55 -0400 Subject: [PATCH 17/37] Make queue GC logging consistent with blob GC logging --- src/queue/gc/QueueGCManager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/queue/gc/QueueGCManager.ts b/src/queue/gc/QueueGCManager.ts index d916c8756..ec58a1726 100644 --- a/src/queue/gc/QueueGCManager.ts +++ b/src/queue/gc/QueueGCManager.ts @@ -145,18 +145,18 @@ export default class QueueGCManager implements IGCManager { private async markSweepLoop(): Promise { while (this._status === Status.Running) { this.logger.info( - `QueueGCManager:markSweepLoop() Start new mark and sweep.` + `QueueGCManager:markSweepLoop() Start next mark and sweep.` ); const start = Date.now(); await this.markSweep(); - const duration = Date.now() - start; + const period = Date.now() - start; this.logger.info( - `QueueGCManager:markSweepLoop() Mark and sweep finished, take ${duration}ms.` + `QueueGCManager:markSweepLoop() Mark and sweep finished, taken ${period}ms.` ); if (this._status === Status.Running) { this.logger.info( - `QueueGCManager:markSweepLoop() Sleep for ${this.gcIntervalInMS}` + `QueueGCManager:markSweepLoop() Sleep for ${this.gcIntervalInMS}ms.` ); await this.sleep(this.gcIntervalInMS); } From d7c707a35da805986543432e70a4e7d9e55df5a8 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 15:43:00 -0400 Subject: [PATCH 18/37] Add VS Code option --- package.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package.json b/package.json index 364296298..25964e72a 100644 --- a/package.json +++ b/package.json @@ -248,6 +248,11 @@ "type": "boolean", "default": false, "description": "Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri." + }, + "azurite.inMemoryPersistence": { + "type": "boolean", + "default": false, + "description": "Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." } } } From 7379048d47f7fd222dfdd6d011afc9b6f10ab267 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 17 Oct 2023 15:47:56 -0400 Subject: [PATCH 19/37] Improve setting description --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 25964e72a..70f3d4302 100644 --- a/package.json +++ b/package.json @@ -252,7 +252,7 @@ "azurite.inMemoryPersistence": { "type": "boolean", "default": false, - "description": "Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + "description": "Disable persisting any data to disk. All data is stored in memory. If the Azurite (node) process is terminated or VS Code is closed, all data is lost." } } } From cda16fb2a60c129b7d6f60de9583bab8fdcdedba Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 18 Oct 2023 10:55:53 -0400 Subject: [PATCH 20/37] Don't include docs in the VS Code extension --- .vscodeignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscodeignore b/.vscodeignore index 6af52f223..e99be99e5 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -30,4 +30,5 @@ temp .github .prettierrc.json README.mcr.md -release \ No newline at end of file +release +docs From a85519d3c6286d1c203745d337c17661c1feed62 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Fri, 20 Oct 2023 11:15:43 -0400 Subject: [PATCH 21/37] Fix merge --- tests/table/apis/table.validation.rest.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/table/apis/table.validation.rest.test.ts b/tests/table/apis/table.validation.rest.test.ts index 8c15fa7c9..524339a4e 100644 --- a/tests/table/apis/table.validation.rest.test.ts +++ b/tests/table/apis/table.validation.rest.test.ts @@ -5,9 +5,8 @@ // special care is needed to replace etags and folders when used import * as assert from "assert"; import { configLogger } from "../../../src/common/Logger"; -import TableConfiguration from "../../../src/table/TableConfiguration"; import TableServer from "../../../src/table/TableServer"; -import { getUniqueName } from "../../testutils"; +import { getUniqueName } from "../../testutils"; import { deleteToAzurite, getToAzurite, @@ -16,6 +15,7 @@ import { getToAzuriteProductionUrl } from "../utils/table.entity.tests.rest.submitter"; import dns = require("dns"); +import TableTestServerFactory from "../utils/TableTestServerFactory"; // Set true to enable debug log configLogger(false); @@ -401,7 +401,7 @@ describe("table name validation tests", () => { } }); - + it(`Should work with production style URL when ${productionStyleHostName} is resolvable`, async () => { await dns.promises.lookup(productionStyleHostName).then( async (lookupAddress) => { @@ -414,7 +414,7 @@ describe("table name validation tests", () => { Accept: "application/json;odata=nometadata" }; try { - let response = await postToAzuriteProductionUrl(productionStyleHostName,"Tables", body, createTableHeaders); + let response = await postToAzuriteProductionUrl(productionStyleHostName, "Tables", body, createTableHeaders); assert.strictEqual(response.status, 201); } catch (err: any) { assert.fail(); @@ -446,9 +446,9 @@ describe("table name validation tests", () => { Accept: "application/json;odata=nometadata" }; try { - let response = await postToAzuriteProductionUrl(productionStyleHostName,"Tables", body, createTableHeaders); + let response = await postToAzuriteProductionUrl(productionStyleHostName, "Tables", body, createTableHeaders); assert.strictEqual(response.status, 201); - let tablesList = await getToAzuriteProductionUrl(productionStyleHostNameForSecondary,"Tables", createTableHeaders); + let tablesList = await getToAzuriteProductionUrl(productionStyleHostNameForSecondary, "Tables", createTableHeaders); assert.strictEqual(tablesList.status, 200); } catch (err: any) { assert.fail(); From a05dfd885e50d25f6678d94f737c0027a122d119 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Fri, 20 Oct 2023 14:43:42 -0400 Subject: [PATCH 22/37] Prevent SQL and in-memory persistence at the same time --- README.md | 3 ++- src/blob/BlobConfiguration.ts | 3 +-- src/blob/BlobServerFactory.ts | 5 ++++- src/blob/SqlBlobConfiguration.ts | 2 -- src/blob/SqlBlobServer.ts | 6 +----- src/common/ConfigurationBase.ts | 1 - src/queue/QueueConfiguration.ts | 3 +-- src/table/TableConfiguration.ts | 3 +-- tests/BlobTestServerFactory.ts | 5 ++++- 9 files changed, 14 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 7991aa55b..550c33750 100644 --- a/README.md +++ b/README.md @@ -437,7 +437,8 @@ Optional. When using FQDN instead of IP in request Uri host, by default Azurite Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. By default, LokiJS persists blob and queue metadata to disk and content to extent files. Table storage -persists all data to disk. This behavior can be disabled using this option. +persists all data to disk. This behavior can be disabled using this option. This setting is rejected when +the SQL based metadata implementation is enabled. ```cmd --inMemoryStorage diff --git a/src/blob/BlobConfiguration.ts b/src/blob/BlobConfiguration.ts index 619b0a399..d3674f2d6 100644 --- a/src/blob/BlobConfiguration.ts +++ b/src/blob/BlobConfiguration.ts @@ -40,7 +40,7 @@ export default class BlobConfiguration extends ConfigurationBase { pwd: string = "", oauth?: string, disableProductStyleUrl: boolean = false, - isMemoryPersistence: boolean = false, + public readonly isMemoryPersistence: boolean = false, ) { super( host, @@ -56,7 +56,6 @@ export default class BlobConfiguration extends ConfigurationBase { pwd, oauth, disableProductStyleUrl, - isMemoryPersistence, ); } } diff --git a/src/blob/BlobServerFactory.ts b/src/blob/BlobServerFactory.ts index 8d5b4a181..e138a185a 100644 --- a/src/blob/BlobServerFactory.ts +++ b/src/blob/BlobServerFactory.ts @@ -42,6 +42,10 @@ export class BlobServerFactory { const isSQL = databaseConnectionString !== undefined; if (isSQL) { + if (env.inMemoryPersistence()) { + throw new Error(`The in-memory persistence settings is not supported when using SQL-based metadata.`) + } + const config = new SqlBlobConfiguration( env.blobHost(), env.blobPort(), @@ -59,7 +63,6 @@ export class BlobServerFactory { env.pwd(), env.oauth(), env.disableProductStyleUrl(), - env.inMemoryPersistence(), ); return new SqlBlobServer(config); diff --git a/src/blob/SqlBlobConfiguration.ts b/src/blob/SqlBlobConfiguration.ts index 831a62335..093175a7b 100644 --- a/src/blob/SqlBlobConfiguration.ts +++ b/src/blob/SqlBlobConfiguration.ts @@ -36,7 +36,6 @@ export default class SqlBlobConfiguration extends ConfigurationBase { pwd: string = "", oauth?: string, disableProductStyleUrl: boolean = false, - isMemoryPersistence: boolean = false, ) { super( host, @@ -52,7 +51,6 @@ export default class SqlBlobConfiguration extends ConfigurationBase { pwd, oauth, disableProductStyleUrl, - isMemoryPersistence, ); } } diff --git a/src/blob/SqlBlobServer.ts b/src/blob/SqlBlobServer.ts index b36055732..c0e07e6d3 100644 --- a/src/blob/SqlBlobServer.ts +++ b/src/blob/SqlBlobServer.ts @@ -8,7 +8,6 @@ import IGCManager from "../common/IGCManager"; import IRequestListenerFactory from "../common/IRequestListenerFactory"; import logger from "../common/Logger"; import FSExtentStore from "../common/persistence/FSExtentStore"; -import MemoryExtentStore from "../common/persistence/MemoryExtentStore"; import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; import IExtentStore from "../common/persistence/IExtentStore"; import SqlExtentMetadataStore from "../common/persistence/SqlExtentMetadataStore"; @@ -77,10 +76,7 @@ export default class SqlBlobServer extends ServerBase { configuration.sequelizeOptions ); - const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore( - extentMetadataStore, - logger - ) : new FSExtentStore( + const extentStore: IExtentStore = new FSExtentStore( extentMetadataStore, configuration.persistenceArray, logger diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index b1f634473..42336eec7 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -22,7 +22,6 @@ export default abstract class ConfigurationBase { public readonly pwd: string = "", public readonly oauth?: string, public readonly disableProductStyleUrl: boolean = false, - public readonly isMemoryPersistence: boolean = false, ) {} public hasCert() { diff --git a/src/queue/QueueConfiguration.ts b/src/queue/QueueConfiguration.ts index 0eed1e2f8..89fa0b3bd 100644 --- a/src/queue/QueueConfiguration.ts +++ b/src/queue/QueueConfiguration.ts @@ -40,7 +40,7 @@ export default class QueueConfiguration extends ConfigurationBase { pwd: string = "", oauth?: string, disableProductStyleUrl: boolean = false, - isMemoryPersistence: boolean = false, + public readonly isMemoryPersistence: boolean = false, ) { super( host, @@ -56,7 +56,6 @@ export default class QueueConfiguration extends ConfigurationBase { pwd, oauth, disableProductStyleUrl, - isMemoryPersistence, ); } } diff --git a/src/table/TableConfiguration.ts b/src/table/TableConfiguration.ts index ac3980385..3fe7308bf 100644 --- a/src/table/TableConfiguration.ts +++ b/src/table/TableConfiguration.ts @@ -36,7 +36,7 @@ export default class TableConfiguration extends ConfigurationBase { pwd: string = "", oauth?: string, disableProductStyleUrl: boolean = false, - isMemoryPersistence: boolean = false, + public readonly isMemoryPersistence: boolean = false, ) { super( host, @@ -52,7 +52,6 @@ export default class TableConfiguration extends ConfigurationBase { pwd, oauth, disableProductStyleUrl, - isMemoryPersistence, ); } } diff --git a/tests/BlobTestServerFactory.ts b/tests/BlobTestServerFactory.ts index 769e32c18..d83b5a4c0 100644 --- a/tests/BlobTestServerFactory.ts +++ b/tests/BlobTestServerFactory.ts @@ -29,6 +29,10 @@ export default class BlobTestServerFactory { const key = https ? "tests/server.key" : undefined; if (isSQL) { + if (inMemoryPersistence) { + throw new Error(`The in-memory persistence settings is not supported when using SQL-based metadata.`) + } + const config = new SqlBlobConfiguration( host, port, @@ -46,7 +50,6 @@ export default class BlobTestServerFactory { undefined, oauth, undefined, - inMemoryPersistence ); return new SqlBlobServer(config); From c371f77d4ee16b080bf4d2a5153e9ca4cf11dcf2 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Fri, 20 Oct 2023 15:23:22 -0400 Subject: [PATCH 23/37] Add MemoryExtentChunkStore for shared in-memory size tracking --- src/common/persistence/MemoryExtentStore.ts | 66 ++++++++++++-- tests/blob/memory.unit.test.ts | 99 +++++++++++++++++++++ 2 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 tests/blob/memory.unit.test.ts diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts index 803d1fb51..abd501df3 100644 --- a/src/common/persistence/MemoryExtentStore.ts +++ b/src/common/persistence/MemoryExtentStore.ts @@ -11,19 +11,73 @@ export interface IMemoryExtentChunk extends IExtentChunk { chunks: (Buffer | string)[] } +export class MemoryExtentChunkStore { + private readonly _sizeLimit: number; + private readonly _chunks: Map = new Map(); + private _totalSize: number = 0; + + public constructor(sizeLimit: number) { + this._sizeLimit = sizeLimit; + } + + public clear(): void { + this._chunks.clear() + this._totalSize = 0 + } + + public set(chunk: IMemoryExtentChunk): void { + let delta = chunk.count + const existing = this._chunks.get(chunk.id) + if (existing) { + delta -= existing.count + } + + if (this._totalSize + delta > this._sizeLimit) { + throw new Error(`Cannot add an extent chunk to the in-memory store. Size limit of ${this._sizeLimit} bytes will be exceeded.`) + } + + this._chunks.set(chunk.id, chunk) + this._totalSize += delta + } + + public get(id: string): IMemoryExtentChunk | undefined { + return this._chunks.get(id) + } + + public delete(id: string): boolean { + const existing = this._chunks.get(id); + if (existing) { + this._chunks.delete(id) + this._totalSize -= existing.count + return true + } + + return false + } + + public totalSize(): number { + return this._totalSize + } + + public sizeLimit(): number { + return this._sizeLimit + } +} + export default class MemoryExtentStore implements IExtentStore { + private readonly chunks: MemoryExtentChunkStore; private readonly metadataStore: IExtentMetadataStore; private readonly logger: ILogger; - private readonly chunks: Map = new Map(); - private totalSize: number = 0; private initialized: boolean = false; private closed: boolean = true; public constructor( + chunks: MemoryExtentChunkStore, metadata: IExtentMetadataStore, logger: ILogger ) { + this.chunks = chunks; this.metadataStore = metadata; this.logger = logger; } @@ -89,8 +143,7 @@ export default class MemoryExtentStore implements IExtentStore { contextId ); - this.chunks.set(extentChunk.id, extentChunk); - this.totalSize += count + this.chunks.set(extentChunk); this.logger.debug( `MemoryExtentStore:appendExtent() Added chunks to in-memory map. id:${extentChunk.id} `, @@ -136,7 +189,7 @@ export default class MemoryExtentStore implements IExtentStore { } this.logger.debug( - `MemoryExtentStore:readExtent() Fetched chunks from in-memory map. id:${match.id} count:${match.count} chunks.length:${match.chunks.length} totalSize:${this.totalSize}`, + `MemoryExtentStore:readExtent() Fetched chunks from in-memory map. id:${match.id} count:${match.count} chunks.length:${match.chunks.length} totalSize:${this.chunks.totalSize()}`, contextId ); @@ -260,11 +313,10 @@ export default class MemoryExtentStore implements IExtentStore { const extent = this.chunks.get(id) if (extent) { this.chunks.delete(id) - this.totalSize -= extent.count } await this.metadataStore.deleteExtent(id); this.logger.debug( - `MemoryExtentStore:deleteExtents() Deleted extent:${id} totalSize:${this.totalSize}` + `MemoryExtentStore:deleteExtents() Deleted extent:${id} totalSize:${this.chunks.totalSize()}` ); count++; } diff --git a/tests/blob/memory.unit.test.ts b/tests/blob/memory.unit.test.ts new file mode 100644 index 000000000..3dfbe33fd --- /dev/null +++ b/tests/blob/memory.unit.test.ts @@ -0,0 +1,99 @@ +import assert = require("assert"); +import { IMemoryExtentChunk, MemoryExtentChunkStore } from "../../src/common/persistence/MemoryExtentStore"; + +function chunk(id: string, count: number, fill?: string): IMemoryExtentChunk { + return { + id, + chunks: [Buffer.alloc(count, fill)], + count: count, + offset: 0, + } +} + +describe("MemoryExtentChunkStore", () => { + + it("should limit max size should work", () => { + const store = new MemoryExtentChunkStore(1000); + store.set(chunk("a", 1000)) + + assert.throws( + () => store.set(chunk("b", 1)), + /Cannot add an extent chunk to the in-memory store. Size limit of 1000 bytes will be exceeded./) + }); + + it("updates current size for add", () => { + const store = new MemoryExtentChunkStore(1000); + + store.set(chunk("a", 555)) + store.set(chunk("b", 1)) + store.set(chunk("c", 123)) + + assert.strictEqual(679, store.totalSize()) + }); + + it("updates current size based on count property", () => { + const store = new MemoryExtentChunkStore(1000); + + store.set({ + id: "a", + chunks: [Buffer.alloc(10, 'a'), Buffer.alloc(20, 'b')], + count: 15, // a lie, for testing + offset: 0 + }) + + assert.strictEqual(15, store.totalSize()) + }); + + it("updates current size for delete", () => { + const store = new MemoryExtentChunkStore(1000); + store.set(chunk("a", 555)) + store.set(chunk("b", 1)) + store.set(chunk("c", 123)) + + store.delete("b") + + assert.strictEqual(678, store.totalSize()) + }); + + it("updates current size with delta when ID is replaced", () => { + const store = new MemoryExtentChunkStore(1000); + store.set(chunk("a", 555)) + store.set(chunk("b", 1)) + + store.set(chunk("a", 123)) + + assert.strictEqual(124, store.totalSize()) + }); + + it("resets current size for clear", () => { + const store = new MemoryExtentChunkStore(1000); + store.set(chunk("a", 555)) + store.set(chunk("b", 1)) + store.set(chunk("c", 123)) + + store.clear() + + assert.strictEqual(0, store.totalSize()) + }); + + it("replaces buffers if ID is existing", () => { + const store = new MemoryExtentChunkStore(1000); + store.set(chunk("a", 11, '0')) + + store.set(chunk("a", 12, '1')) + + const existing = store.get('a') + assert.strictEqual(1, existing?.chunks.length) + assert.deepStrictEqual(Buffer.alloc(12, '1'), existing?.chunks[0]) + }); + + it("allows deletion by ID", () => { + const store = new MemoryExtentChunkStore(1000); + store.set(chunk("a", 11, '0')) + + store.delete("a") + + const existing = store.get('a') + assert.strictEqual(undefined, existing) + }); +}); From 350dee7cee1e43864590c60511e30b830d8122b0 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Fri, 20 Oct 2023 17:11:07 -0400 Subject: [PATCH 24/37] Use a shared MemoryExtentChunkStore by default --- src/blob/BlobConfiguration.ts | 2 ++ src/blob/BlobServer.ts | 3 +- src/common/persistence/MemoryExtentStore.ts | 24 +++++++++++++--- src/queue/QueueConfiguration.ts | 2 ++ src/queue/QueueServer.ts | 3 +- ....unit.test.ts => memoryStore.unit.test.ts} | 28 ++++++++++++++++++- 6 files changed, 55 insertions(+), 7 deletions(-) rename tests/blob/{memory.unit.test.ts => memoryStore.unit.test.ts} (69%) diff --git a/src/blob/BlobConfiguration.ts b/src/blob/BlobConfiguration.ts index d3674f2d6..cc31fa5a9 100644 --- a/src/blob/BlobConfiguration.ts +++ b/src/blob/BlobConfiguration.ts @@ -1,5 +1,6 @@ import ConfigurationBase from "../common/ConfigurationBase"; import { StoreDestinationArray } from "../common/persistence/IExtentStore"; +import { MemoryExtentChunkStore } from "../common/persistence/MemoryExtentStore"; import { DEFAULT_BLOB_EXTENT_LOKI_DB_PATH, DEFAULT_BLOB_LISTENING_PORT, @@ -41,6 +42,7 @@ export default class BlobConfiguration extends ConfigurationBase { oauth?: string, disableProductStyleUrl: boolean = false, public readonly isMemoryPersistence: boolean = false, + public readonly memoryStore?: MemoryExtentChunkStore, ) { super( host, diff --git a/src/blob/BlobServer.ts b/src/blob/BlobServer.ts index 9c8d3439d..643485949 100644 --- a/src/blob/BlobServer.ts +++ b/src/blob/BlobServer.ts @@ -9,7 +9,7 @@ import IGCManager from "../common/IGCManager"; import IRequestListenerFactory from "../common/IRequestListenerFactory"; import logger from "../common/Logger"; import FSExtentStore from "../common/persistence/FSExtentStore"; -import MemoryExtentStore from "../common/persistence/MemoryExtentStore"; +import MemoryExtentStore, { SharedChunkStore } from "../common/persistence/MemoryExtentStore"; import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; import IExtentStore from "../common/persistence/IExtentStore"; import LokiExtentMetadataStore from "../common/persistence/LokiExtentMetadataStore"; @@ -84,6 +84,7 @@ export default class BlobServer extends ServerBase implements ICleaner { ); const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore( + configuration.memoryStore ?? SharedChunkStore, extentMetadataStore, logger ) : new FSExtentStore( diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts index abd501df3..6e0cbc4e0 100644 --- a/src/common/persistence/MemoryExtentStore.ts +++ b/src/common/persistence/MemoryExtentStore.ts @@ -6,17 +6,18 @@ import IExtentStore, { IExtentChunk } from "./IExtentStore"; import uuid = require("uuid"); import multistream = require("multistream"); import { Readable } from "stream"; +import { totalmem } from "os"; export interface IMemoryExtentChunk extends IExtentChunk { chunks: (Buffer | string)[] } export class MemoryExtentChunkStore { - private readonly _sizeLimit: number; + private _sizeLimit?: number; private readonly _chunks: Map = new Map(); private _totalSize: number = 0; - public constructor(sizeLimit: number) { + public constructor(sizeLimit?: number) { this._sizeLimit = sizeLimit; } @@ -32,7 +33,7 @@ export class MemoryExtentChunkStore { delta -= existing.count } - if (this._totalSize + delta > this._sizeLimit) { + if (this._sizeLimit != undefined && this._totalSize + delta > this._sizeLimit) { throw new Error(`Cannot add an extent chunk to the in-memory store. Size limit of ${this._sizeLimit} bytes will be exceeded.`) } @@ -59,11 +60,26 @@ export class MemoryExtentChunkStore { return this._totalSize } - public sizeLimit(): number { + public setSizeLimit(sizeLimit?: number) { + if (sizeLimit && sizeLimit < this._totalSize) { + return false; + } + + this._sizeLimit = sizeLimit + return true; + } + + public sizeLimit(): number | undefined { return this._sizeLimit } } +// By default, allow up to half of the total memory to be used for in-memory +// extents. We don't use freemem (free memory instead of total memory) since +// that would lead to a decent amount of unpredictability. +const defaultSize = Math.trunc(totalmem() * 0.5) +export const SharedChunkStore: MemoryExtentChunkStore = new MemoryExtentChunkStore(defaultSize); + export default class MemoryExtentStore implements IExtentStore { private readonly chunks: MemoryExtentChunkStore; private readonly metadataStore: IExtentMetadataStore; diff --git a/src/queue/QueueConfiguration.ts b/src/queue/QueueConfiguration.ts index 89fa0b3bd..b98be78c9 100644 --- a/src/queue/QueueConfiguration.ts +++ b/src/queue/QueueConfiguration.ts @@ -1,5 +1,6 @@ import ConfigurationBase from "../common/ConfigurationBase"; import { StoreDestinationArray } from "../common/persistence/IExtentStore"; +import { MemoryExtentChunkStore } from "../common/persistence/MemoryExtentStore"; import { DEFAULT_ENABLE_ACCESS_LOG, DEFAULT_ENABLE_DEBUG_LOG, @@ -41,6 +42,7 @@ export default class QueueConfiguration extends ConfigurationBase { oauth?: string, disableProductStyleUrl: boolean = false, public readonly isMemoryPersistence: boolean = false, + public readonly memoryStore?: MemoryExtentChunkStore, ) { super( host, diff --git a/src/queue/QueueServer.ts b/src/queue/QueueServer.ts index 2e3bab020..40ecf8aa6 100644 --- a/src/queue/QueueServer.ts +++ b/src/queue/QueueServer.ts @@ -8,7 +8,7 @@ import IGCManager from "../common/IGCManager"; import IRequestListenerFactory from "../common/IRequestListenerFactory"; import logger from "../common/Logger"; import FSExtentStore from "../common/persistence/FSExtentStore"; -import MemoryExtentStore from "../common/persistence/MemoryExtentStore"; +import MemoryExtentStore, { SharedChunkStore } from "../common/persistence/MemoryExtentStore"; import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; import IExtentStore from "../common/persistence/IExtentStore"; import LokiExtentMetadataStore from "../common/persistence/LokiExtentMetadataStore"; @@ -83,6 +83,7 @@ export default class QueueServer extends ServerBase { ); const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore( + configuration.memoryStore ?? SharedChunkStore, extentMetadataStore, logger ) : new FSExtentStore( diff --git a/tests/blob/memory.unit.test.ts b/tests/blob/memoryStore.unit.test.ts similarity index 69% rename from tests/blob/memory.unit.test.ts rename to tests/blob/memoryStore.unit.test.ts index 3dfbe33fd..185d8018c 100644 --- a/tests/blob/memory.unit.test.ts +++ b/tests/blob/memoryStore.unit.test.ts @@ -1,5 +1,6 @@ import assert = require("assert"); -import { IMemoryExtentChunk, MemoryExtentChunkStore } from "../../src/common/persistence/MemoryExtentStore"; +import { IMemoryExtentChunk, MemoryExtentChunkStore, SharedChunkStore } from "../../src/common/persistence/MemoryExtentStore"; +import { totalmem } from "os"; function chunk(id: string, count: number, fill?: string): IMemoryExtentChunk { return { @@ -55,6 +56,25 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(678, store.totalSize()) }); + it("allows size limit to be updated", () => { + const store = new MemoryExtentChunkStore(1000); + store.set(chunk("a", 20)) + + store.setSizeLimit(50) + + assert.throws( + () => store.set(chunk("b", 31)), + /Cannot add an extent chunk to the in-memory store. Size limit of 50 bytes will be exceeded./) + }); + + it("prevents size limit from being set lower than the current size", () => { + const store = new MemoryExtentChunkStore(1000); + store.set(chunk("a", 20)) + + assert.strictEqual(false, store.setSizeLimit(19)) + assert.strictEqual(1000, store.sizeLimit()) + }); + it("updates current size with delta when ID is replaced", () => { const store = new MemoryExtentChunkStore(1000); store.set(chunk("a", 555)) @@ -96,4 +116,10 @@ describe("MemoryExtentChunkStore", () => { const existing = store.get('a') assert.strictEqual(undefined, existing) }); + + it("should have a shared instance defaulting to close to 50% of the total bytes", () => { + assert.ok(SharedChunkStore.sizeLimit(), "The default store's size limit should be set.") + assert.ok(SharedChunkStore.sizeLimit()! > 0.4 * totalmem()) + assert.ok(SharedChunkStore.sizeLimit()! < 0.6 * totalmem()) + }); }); From 057df968591b80ff68372a64684cf9eb73946112 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 25 Oct 2023 18:34:54 -0400 Subject: [PATCH 25/37] Align Table and Queue cleaning approach with Blob This clean-up for in-memory queue extents as well as fixes broken Table clean-up in VS Code --- src/common/VSCServerManagerQueue.ts | 12 ++----- src/queue/QueueServer.ts | 31 ++++++++++++++++++- src/queue/persistence/IQueueMetadataStore.ts | 3 +- .../persistence/LokiQueueMetadataStore.ts | 24 +++++++++++--- src/table/TableServer.ts | 16 +++++++++- src/table/persistence/ITableMetadataStore.ts | 3 +- .../persistence/LokiTableMetadataStore.ts | 16 ++++++++++ 7 files changed, 87 insertions(+), 18 deletions(-) diff --git a/src/common/VSCServerManagerQueue.ts b/src/common/VSCServerManagerQueue.ts index 2529ab7da..791759cf3 100644 --- a/src/common/VSCServerManagerQueue.ts +++ b/src/common/VSCServerManagerQueue.ts @@ -1,6 +1,5 @@ import { access, ensureDir } from "fs-extra"; import { join } from "path"; -import { promisify } from "util"; import QueueConfiguration from "../queue/QueueConfiguration"; import QueueServer from "../queue/QueueServer"; @@ -18,9 +17,6 @@ import VSCEnvironment from "./VSCEnvironment"; import VSCServerManagerBase from "./VSCServerManagerBase"; import VSCServerManagerClosedState from "./VSCServerManagerClosedState"; -import rimraf = require("rimraf"); -const rimrafAsync = promisify(rimraf); - export default class VSCServerManagerBlob extends VSCServerManagerBase { public readonly accessChannelStream = new VSCChannelWriteStream( "Azurite Queue" @@ -62,12 +58,8 @@ export default class VSCServerManagerBlob extends VSCServerManagerBase { } public async cleanImpl(): Promise { - const config = await this.getConfiguration(); - await rimrafAsync(config.extentDBPath); - await rimrafAsync(config.metadataDBPath); - for (const path of config.persistencePathArray) { - await rimrafAsync(path.locationPath); - } + await this.createImpl(); + await this.server!.clean(); } private async getConfiguration(): Promise { diff --git a/src/queue/QueueServer.ts b/src/queue/QueueServer.ts index 40ecf8aa6..385799d69 100644 --- a/src/queue/QueueServer.ts +++ b/src/queue/QueueServer.ts @@ -12,7 +12,7 @@ import MemoryExtentStore, { SharedChunkStore } from "../common/persistence/Memor import IExtentMetadataStore from "../common/persistence/IExtentMetadataStore"; import IExtentStore from "../common/persistence/IExtentStore"; import LokiExtentMetadataStore from "../common/persistence/LokiExtentMetadataStore"; -import ServerBase from "../common/ServerBase"; +import ServerBase, { ServerStatus } from "../common/ServerBase"; import QueueGCManager from "./gc/QueueGCManager"; import IQueueMetadataStore from "./persistence/IQueueMetadataStore"; import LokiQueueMetadataStore from "./persistence/LokiQueueMetadataStore"; @@ -134,6 +134,35 @@ export default class QueueServer extends ServerBase { this.gcManager = gcManager; } + /** + * Clean up server persisted data, including Loki metadata database file, + * Loki extent database file and extent data. + * + * @returns {Promise} + * @memberof BlobServer + */ + public async clean(): Promise { + if (this.getStatus() === ServerStatus.Closed) { + if (this.extentStore !== undefined) { + await this.extentStore.clean(); + } + + if (this.extentMetadataStore !== undefined) { + await this.extentMetadataStore.clean(); + } + + if (this.metadataStore !== undefined) { + await this.metadataStore.clean(); + } + + if (this.accountDataStore !== undefined) { + await this.accountDataStore.clean(); + } + return; + } + throw Error(`Cannot clean up queue server in status ${this.getStatus()}.`); + } + protected async beforeStart(): Promise { const msg = `Azurite Queue service is starting on ${this.host}:${this.port}`; logger.info(msg); diff --git a/src/queue/persistence/IQueueMetadataStore.ts b/src/queue/persistence/IQueueMetadataStore.ts index 7a9a34c5b..f193e7609 100644 --- a/src/queue/persistence/IQueueMetadataStore.ts +++ b/src/queue/persistence/IQueueMetadataStore.ts @@ -1,3 +1,4 @@ +import ICleaner from "../../common/ICleaner"; import IDataStore from "../../common/IDataStore"; import IGCExtentProvider from "../../common/IGCExtentProvider"; import * as Models from "../generated/artifacts/models"; @@ -71,7 +72,7 @@ export interface IExtentChunk { * @interface IQueueMetadataStore * @extends {IDataStore} */ -export interface IQueueMetadataStore extends IGCExtentProvider, IDataStore { +export interface IQueueMetadataStore extends IGCExtentProvider, IDataStore, ICleaner { /** * Update queue service properties. Create service properties document if not exists in persistency layer. * Assume service properties collection has been created during start method. diff --git a/src/queue/persistence/LokiQueueMetadataStore.ts b/src/queue/persistence/LokiQueueMetadataStore.ts index dea44ad49..0cd285172 100644 --- a/src/queue/persistence/LokiQueueMetadataStore.ts +++ b/src/queue/persistence/LokiQueueMetadataStore.ts @@ -14,6 +14,7 @@ import { ServicePropertiesModel } from "./IQueueMetadataStore"; import QueueReferredExtentsAsyncIterator from "./QueueReferredExtentsAsyncIterator"; +import { rimrafAsync } from "../../common/utils/utils"; /** * This is a metadata source implementation for queue based on loki DB. @@ -146,6 +147,21 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore { this.closed = true; } + /** + * Clean LokiQueueMetadataStore. + * + * @returns {Promise} + * @memberof LokiQueueMetadataStore + */ + public async clean(): Promise { + if (this.isClosed()) { + await rimrafAsync(this.lokiDBPath); + + return; + } + throw new Error(`Cannot clean LokiQueueMetadataStore, it's not closed.`); + } + /** * Update queue service properties. Create service properties document if not exists in persistency layer. * Assume service properties collection has been created during start method. @@ -220,10 +236,10 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore { prefix === "" ? { $loki: { $gt: marker }, accountName: account } : { - name: { $regex: `^${this.escapeRegex(prefix)}` }, - $loki: { $gt: marker }, - accountName: account - }; + name: { $regex: `^${this.escapeRegex(prefix)}` }, + $loki: { $gt: marker }, + accountName: account + }; // Get one more item to help check if the query reach the tail of the collection. const docs = coll diff --git a/src/table/TableServer.ts b/src/table/TableServer.ts index 15975b99d..43e7e3126 100644 --- a/src/table/TableServer.ts +++ b/src/table/TableServer.ts @@ -9,7 +9,7 @@ import logger from "../common/Logger"; import ITableMetadataStore from "../table/persistence/ITableMetadataStore"; import LokiTableMetadataStore from "../table/persistence/LokiTableMetadataStore"; -import ServerBase from "../common/ServerBase"; +import ServerBase, { ServerStatus } from "../common/ServerBase"; import TableConfiguration from "./TableConfiguration"; import TableRequestListenerFactory from "./TableRequestListenerFactory"; @@ -74,6 +74,20 @@ export default class TableServer extends ServerBase { this.accountDataStore = accountDataStore; } + public async clean(): Promise { + if (this.getStatus() === ServerStatus.Closed) { + if (this.metadataStore !== undefined) { + await this.metadataStore.clean(); + } + + if (this.accountDataStore !== undefined) { + await this.accountDataStore.clean(); + } + return; + } + throw Error(`Cannot clean up table server in status ${this.getStatus()}.`); + } + protected async beforeStart(): Promise { const msg = `Azurite Table service is starting on ${this.host}:${this.port}`; logger.info(msg); diff --git a/src/table/persistence/ITableMetadataStore.ts b/src/table/persistence/ITableMetadataStore.ts index a414ef7c9..8e5f8fde6 100644 --- a/src/table/persistence/ITableMetadataStore.ts +++ b/src/table/persistence/ITableMetadataStore.ts @@ -1,3 +1,4 @@ +import ICleaner from "../../common/ICleaner"; import * as Models from "../generated/artifacts/models"; import Context from "../generated/Context"; /** MODELS FOR SERVICE */ @@ -45,7 +46,7 @@ export interface IEntity { export type Entity = IEntity & IOdataAnnotationsOptional; -export default interface ITableMetadataStore { +export default interface ITableMetadataStore extends ICleaner { createTable(context: Context, tableModel: Table): Promise; queryTable( context: Context, diff --git a/src/table/persistence/LokiTableMetadataStore.ts b/src/table/persistence/LokiTableMetadataStore.ts index 92b41b171..55186ee5d 100644 --- a/src/table/persistence/LokiTableMetadataStore.ts +++ b/src/table/persistence/LokiTableMetadataStore.ts @@ -9,6 +9,7 @@ import { Entity, Table } from "../persistence/ITableMetadataStore"; import { ODATA_TYPE, QUERY_RESULT_MAX_NUM } from "../utils/constants"; import ITableMetadataStore, { TableACL } from "./ITableMetadataStore"; import LokiTableStoreQueryGenerator from "./LokiTableStoreQueryGenerator"; +import { rimrafAsync } from "../../common/utils/utils"; /** MODELS FOR SERVICE */ interface IServiceAdditionalProperties { @@ -76,6 +77,21 @@ export default class LokiTableMetadataStore implements ITableMetadataStore { this.closed = true; } + /** + * Clean LokiTableMetadataStore. + * + * @returns {Promise} + * @memberof LokiTableMetadataStore + */ + public async clean(): Promise { + if (this.isClosed()) { + await rimrafAsync(this.lokiDBPath); + + return; + } + throw new Error(`Cannot clean LokiTableMetadataStore, it's not closed.`); + } + public isInitialized(): boolean { return this.initialized; } From a028724e8d0547f83ee4caa6e6ef09b72684ce4a Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 25 Oct 2023 18:36:24 -0400 Subject: [PATCH 26/37] Add extentMemoryLimit option per review feedback --- README.md | 28 ++++- package.json | 8 ++ src/blob/BlobEnvironment.ts | 8 ++ src/blob/BlobServer.ts | 5 +- src/blob/BlobServerFactory.ts | 9 +- src/blob/IBlobEnvironment.ts | 1 + src/common/ConfigurationBase.ts | 40 +++++++- src/common/Environment.ts | 12 ++- src/common/VSCEnvironment.ts | 4 + src/common/VSCServerManagerClosedState.ts | 8 ++ src/common/persistence/MemoryExtentStore.ts | 98 +++++++++++++----- src/queue/IQueueEnvironment.ts | 1 + src/queue/QueueEnvironment.ts | 8 ++ src/queue/QueueServer.ts | 5 +- src/queue/main.ts | 3 + src/table/main.ts | 4 +- tests/blob/memoryStore.unit.test.ts | 107 ++++++++++++++------ 17 files changed, 279 insertions(+), 70 deletions(-) diff --git a/README.md b/README.md index 550c33750..3c9c81786 100644 --- a/README.md +++ b/README.md @@ -435,15 +435,35 @@ Optional. When using FQDN instead of IP in request Uri host, by default Azurite ### Use in-memory storage -Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. -By default, LokiJS persists blob and queue metadata to disk and content to extent files. Table storage -persists all data to disk. This behavior can be disabled using this option. This setting is rejected when -the SQL based metadata implementation is enabled. +Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. By default, +LokiJS persists blob and queue metadata to disk and content to extent files. Table storage persists all data to disk. +This behavior can be disabled using this option. This setting is rejected when the SQL based metadata implementation is +enabled. ```cmd --inMemoryStorage ``` +By default, the in-memory extent store (for blob and queue content) is limited to 50% of the total memory on the host +machine. This is evaluated to using [`os.totalmem()`](https://nodejs.org/api/os.html#ostotalmem). This limit can be +overridden using the `--extentMemoryLimit ` option. There is no restriction on the value specified for this +option but virtual memory may be used if the limit exceeds the amount of available physical memory as provided by the +operating system. A high limit may eventually lead to out of memory errors or reduced performance. + +The queue and blob extent storage count towards the same limit. The `--extentMemoryLimit` setting is rejected when +`--inMemoryStorage` is not specified. LokiJS storage (blob and queue metadata and table metadata and content) do not +contribute to this limit and are unbounded which is the same as without the `--inMemoryStorage` option. + +```cmd +--extentMemoryLimit +``` + +When the limit is reached, write operations to the blob or queue endpoints which carry content will fail with an `HTTP +409` status code, a custom storage error code of `MemoryExtentStoreAtSizeLimit`, and a helpful error message. +Well-behaved storage SDKs and tools will not a retry on this failure and will return a related error message. If this +error is met, consider deleting some in-memory content (blobs or queues), raising the limit, or restarting the Azurite +server thus resetting the storage completely. + ### Command Line Options Differences between Azurite V2 Azurite V3 supports SharedKey, Account Shared Access Signature (SAS), Service SAS, OAuth, and Public Container Access authentications, you can use any Azure Storage SDKs or tools like Storage Explorer to connect Azurite V3 with any authentication strategy. diff --git a/package.json b/package.json index 6c3904b2c..4f7bd528e 100644 --- a/package.json +++ b/package.json @@ -253,6 +253,14 @@ "type": "boolean", "default": false, "description": "Disable persisting any data to disk. All data is stored in memory. If the Azurite (node) process is terminated or VS Code is closed, all data is lost." + }, + "azurite.extentMemoryLimit": { + "type": [ + "number", + "null" + ], + "default": null, + "description": "When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of bytes. Defaults to 50% of total memory." } } } diff --git a/src/blob/BlobEnvironment.ts b/src/blob/BlobEnvironment.ts index ee82d9e33..75122e66e 100644 --- a/src/blob/BlobEnvironment.ts +++ b/src/blob/BlobEnvironment.ts @@ -44,6 +44,10 @@ if (!(args as any).config.name) { ["", "inMemoryPersistence"], "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." ) + .option( + ["", "extentMemoryLimit"], + "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", + ) .option( ["d", "debug"], "Optional. Enable debug log by providing a valid local file path as log destination" @@ -129,6 +133,10 @@ export default class BlobEnvironment implements IBlobEnvironment { return false; } + public extentMemoryLimit(): number | undefined { + return this.flags.extentMemoryLimit; + } + public async debug(): Promise { if (typeof this.flags.debug === "string") { // Enable debug log to file diff --git a/src/blob/BlobServer.ts b/src/blob/BlobServer.ts index 643485949..2f0e10dc3 100644 --- a/src/blob/BlobServer.ts +++ b/src/blob/BlobServer.ts @@ -19,6 +19,7 @@ import BlobRequestListenerFactory from "./BlobRequestListenerFactory"; import BlobGCManager from "./gc/BlobGCManager"; import IBlobMetadataStore from "./persistence/IBlobMetadataStore"; import LokiBlobMetadataStore from "./persistence/LokiBlobMetadataStore"; +import StorageError from "./errors/StorageError"; const BEFORE_CLOSE_MESSAGE = `Azurite Blob service is closing...`; const BEFORE_CLOSE_MESSAGE_GC_ERROR = `Azurite Blob service is closing... Critical error happens during GC.`; @@ -84,9 +85,11 @@ export default class BlobServer extends ServerBase implements ICleaner { ); const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore( + "blob", configuration.memoryStore ?? SharedChunkStore, extentMetadataStore, - logger + logger, + (sc, er, em, ri) => new StorageError(sc, er, em, ri) ) : new FSExtentStore( extentMetadataStore, configuration.persistencePathArray, diff --git a/src/blob/BlobServerFactory.ts b/src/blob/BlobServerFactory.ts index e138a185a..1ea4f8455 100644 --- a/src/blob/BlobServerFactory.ts +++ b/src/blob/BlobServerFactory.ts @@ -13,6 +13,7 @@ import { DEFAULT_BLOB_LOKI_DB_PATH, DEFAULT_BLOB_PERSISTENCE_ARRAY } from "./utils/constants"; +import { setExtentMemoryLimit } from "../common/ConfigurationBase"; export class BlobServerFactory { public async createServer( @@ -43,7 +44,10 @@ export class BlobServerFactory { if (isSQL) { if (env.inMemoryPersistence()) { - throw new Error(`The in-memory persistence settings is not supported when using SQL-based metadata.`) + throw new Error(`The --inMemoryPersistence option is not supported when using SQL-based metadata storage.`) + } + if (env.extentMemoryLimit() !== undefined) { + throw new Error(`The --extentMemoryLimit option is not supported when using SQL-based metadata storage.`) } const config = new SqlBlobConfiguration( @@ -86,6 +90,9 @@ export class BlobServerFactory { env.disableProductStyleUrl(), env.inMemoryPersistence(), ); + + setExtentMemoryLimit(env); + return new BlobServer(config); } } else { diff --git a/src/blob/IBlobEnvironment.ts b/src/blob/IBlobEnvironment.ts index 06bd7342b..b83cddbd3 100644 --- a/src/blob/IBlobEnvironment.ts +++ b/src/blob/IBlobEnvironment.ts @@ -12,4 +12,5 @@ export default interface IBlobEnvironment { oauth(): string | undefined; disableProductStyleUrl(): boolean; inMemoryPersistence(): boolean; + extentMemoryLimit(): number | undefined; } diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index 42336eec7..3c2e209f2 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -1,5 +1,9 @@ import * as fs from "fs"; import { OAuthLevel } from "./models"; +import IBlobEnvironment from "../blob/IBlobEnvironment"; +import IQueueEnvironment from "../queue/IQueueEnvironment"; +import { SharedChunkStore } from "./persistence/MemoryExtentStore"; +import { totalmem } from "os"; export enum CertOptions { Default, @@ -7,6 +11,35 @@ export enum CertOptions { PFX } +export function setExtentMemoryLimit(env: IBlobEnvironment | IQueueEnvironment) { + if (env.inMemoryPersistence()) { + let limit = env.extentMemoryLimit() ?? SharedChunkStore.sizeLimit(); + if (limit && limit >= 0) { + const kb = limit / 1024; + const mb = kb / 1024; + const gb = mb / 1024; + let display; + if (gb >= 1) { + display = `${gb.toFixed(2)} GB` + } else if (mb >= 1) { + display = `${mb.toFixed(2)} MB` + } else { + display = `${kb.toFixed(2)} KB` + } + + const totalPct = Math.round(100 * limit / totalmem()) + console.log(`In-memory extent storage is enabled with a limit of ${display} (${limit} bytes, ${totalPct}% of total memory).`); + SharedChunkStore.setSizeLimit(limit); + } else { + console.log(`In-memory storage is enabled with no limit on memory used.`); + } + } else { + if (env.extentMemoryLimit() !== undefined) { + throw new Error(`The --extentMemoryLimit option is only supported when the --inMemoryPersistence option is set.`) + } + } +} + export default abstract class ConfigurationBase { public constructor( public readonly host: string, @@ -22,7 +55,7 @@ export default abstract class ConfigurationBase { public readonly pwd: string = "", public readonly oauth?: string, public readonly disableProductStyleUrl: boolean = false, - ) {} + ) { } public hasCert() { if (this.cert.length > 0 && this.key.length > 0) { @@ -63,8 +96,7 @@ export default abstract class ConfigurationBase { } public getHttpServerAddress(): string { - return `http${this.hasCert() === CertOptions.Default ? "" : "s"}://${ - this.host - }:${this.port}`; + return `http${this.hasCert() === CertOptions.Default ? "" : "s"}://${this.host + }:${this.port}`; } } diff --git a/src/common/Environment.ts b/src/common/Environment.ts index 3d2968710..49994fdb2 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -64,7 +64,7 @@ args ) .option( ["", "disableProductStyleUrl"], - "Optional. Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri." + "Optional. Disable getting account name from the host of request Uri, always get account name from the first path segment of request Uri" ) .option(["", "oauth"], 'Optional. OAuth level. Candidate values: "basic"') .option(["", "cert"], "Optional. Path to certificate file") @@ -72,7 +72,11 @@ args .option(["", "pwd"], "Optional. Password for .pfx file") .option( ["", "inMemoryPersistence"], - "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost" + ) + .option( + ["", "extentMemoryLimit"], + "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", ) .option( ["d", "debug"], @@ -166,6 +170,10 @@ export default class Environment implements IEnvironment { return false; } + public extentMemoryLimit(): number | undefined { + return this.flags.extentMemoryLimit; + } + public async debug(): Promise { if (typeof this.flags.debug === "string") { // Enable debug log to file diff --git a/src/common/VSCEnvironment.ts b/src/common/VSCEnvironment.ts index 246ea364a..f5ea1fa4d 100644 --- a/src/common/VSCEnvironment.ts +++ b/src/common/VSCEnvironment.ts @@ -113,4 +113,8 @@ export default class VSCEnvironment implements IEnvironment { public inMemoryPersistence(): boolean { return this.workspaceConfiguration.get("inMemoryPersistence") || false; } + + public extentMemoryLimit(): number | undefined { + return this.workspaceConfiguration.get("extentMemoryLimit"); + } } diff --git a/src/common/VSCServerManagerClosedState.ts b/src/common/VSCServerManagerClosedState.ts index dda2bd473..82daaa5b5 100644 --- a/src/common/VSCServerManagerClosedState.ts +++ b/src/common/VSCServerManagerClosedState.ts @@ -1,6 +1,8 @@ import IVSCServerManagerState from "./IVSCServerManagerState"; +import VSCEnvironment from "./VSCEnvironment"; import VSCServerManagerBase from "./VSCServerManagerBase"; import { VSCServerManagerRunningState } from "./VSCServerManagerRunningState"; +import { SharedChunkStore } from "./persistence/MemoryExtentStore"; export default class VSCServerManagerClosedState implements IVSCServerManagerState { @@ -9,6 +11,12 @@ export default class VSCServerManagerClosedState ): Promise { await serverManager.createImpl(); await serverManager.startImpl(); + + const limit = new VSCEnvironment().extentMemoryLimit() + if (limit) { + SharedChunkStore.setSizeLimit(limit) + } + return new VSCServerManagerRunningState(); } diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts index 6e0cbc4e0..a9ae36e6a 100644 --- a/src/common/persistence/MemoryExtentStore.ts +++ b/src/common/persistence/MemoryExtentStore.ts @@ -12,48 +12,87 @@ export interface IMemoryExtentChunk extends IExtentChunk { chunks: (Buffer | string)[] } +interface IExtentCategoryChunks { + chunks: Map, + totalSize: number, +} + export class MemoryExtentChunkStore { private _sizeLimit?: number; - private readonly _chunks: Map = new Map(); + + private readonly _chunks: Map = new Map(); private _totalSize: number = 0; public constructor(sizeLimit?: number) { this._sizeLimit = sizeLimit; } - public clear(): void { - this._chunks.clear() - this._totalSize = 0 + public clear(categoryName: string): void { + let category = this._chunks.get(categoryName) + if (!category) { + return + } + + this._totalSize -= category.totalSize + this._chunks.delete(categoryName) } - public set(chunk: IMemoryExtentChunk): void { + public set(categoryName: string, chunk: IMemoryExtentChunk) { + if (!this.trySet(categoryName, chunk)) { + throw new Error(`Cannot add an extent chunk to the in-memory store. Size limit of ${this._sizeLimit} bytes will be exceeded.`) + } + } + + public trySet(categoryName: string, chunk: IMemoryExtentChunk): boolean { + let category = this._chunks.get(categoryName) + if (!category) { + category = { + chunks: new Map(), + totalSize: 0 + } + this._chunks.set(categoryName, category) + } + let delta = chunk.count - const existing = this._chunks.get(chunk.id) + const existing = category.chunks.get(chunk.id) if (existing) { delta -= existing.count } if (this._sizeLimit != undefined && this._totalSize + delta > this._sizeLimit) { - throw new Error(`Cannot add an extent chunk to the in-memory store. Size limit of ${this._sizeLimit} bytes will be exceeded.`) + return false } - this._chunks.set(chunk.id, chunk) + category.chunks.set(chunk.id, chunk) + category.totalSize += delta this._totalSize += delta + return true } - public get(id: string): IMemoryExtentChunk | undefined { - return this._chunks.get(id) + public get(categoryName: string, id: string): IMemoryExtentChunk | undefined { + return this._chunks.get(categoryName)?.chunks.get(id) } - public delete(id: string): boolean { - const existing = this._chunks.get(id); - if (existing) { - this._chunks.delete(id) - this._totalSize -= existing.count - return true + public delete(categoryName: string, id: string): boolean { + const category = this._chunks.get(categoryName); + if (!category) { + return false + } + + const existing = category.chunks.get(id); + if (!existing) { + return false } - return false + category.chunks.delete(id) + category.totalSize -= existing.count + this._totalSize -= existing.count + + if (category.chunks.size === 0) { + this._chunks.delete(categoryName) + } + + return true } public totalSize(): number { @@ -81,21 +120,26 @@ const defaultSize = Math.trunc(totalmem() * 0.5) export const SharedChunkStore: MemoryExtentChunkStore = new MemoryExtentChunkStore(defaultSize); export default class MemoryExtentStore implements IExtentStore { + private readonly categoryName: string; private readonly chunks: MemoryExtentChunkStore; private readonly metadataStore: IExtentMetadataStore; private readonly logger: ILogger; + private readonly makeError: (statusCode: number, storageErrorCode: string, storageErrorMessage: string, storageRequestID: string) => Error; private initialized: boolean = false; private closed: boolean = true; - public constructor( + categoryName: string, chunks: MemoryExtentChunkStore, metadata: IExtentMetadataStore, - logger: ILogger + logger: ILogger, + makeError: (statusCode: number, storageErrorCode: string, storageErrorMessage: string, storageRequestID: string) => Error, ) { + this.categoryName = categoryName; this.chunks = chunks; this.metadataStore = metadata; this.logger = logger; + this.makeError = makeError; } public isInitialized(): boolean { @@ -125,7 +169,7 @@ export default class MemoryExtentStore implements IExtentStore { public async clean(): Promise { if (this.isClosed()) { - this.chunks.clear(); + this.chunks.clear(this.categoryName); return; } throw new Error(`Cannot clean MemoryExtentStore, it's not closed.`); @@ -159,7 +203,13 @@ export default class MemoryExtentStore implements IExtentStore { contextId ); - this.chunks.set(extentChunk); + if (!this.chunks.trySet(this.categoryName, extentChunk)) { + throw this.makeError( + 409, + "MemoryExtentStoreAtSizeLimit", + `Cannot add an extent chunk to the in-memory store. Size limit of ${this.chunks.sizeLimit()} bytes will be exceeded`, + contextId ?? ""); + } this.logger.debug( `MemoryExtentStore:appendExtent() Added chunks to in-memory map. id:${extentChunk.id} `, @@ -199,7 +249,7 @@ export default class MemoryExtentStore implements IExtentStore { contextId ); - const match = this.chunks.get(extentChunk.id); + const match = this.chunks.get(this.categoryName, extentChunk.id); if (!match) { throw new Error(`Extend ${extentChunk.id} does not exist.`); } @@ -326,9 +376,9 @@ export default class MemoryExtentStore implements IExtentStore { this.logger.info( `MemoryExtentStore:deleteExtents() Delete extent:${id}` ); - const extent = this.chunks.get(id) + const extent = this.chunks.get(this.categoryName, id) if (extent) { - this.chunks.delete(id) + this.chunks.delete(this.categoryName, id) } await this.metadataStore.deleteExtent(id); this.logger.debug( diff --git a/src/queue/IQueueEnvironment.ts b/src/queue/IQueueEnvironment.ts index 7d2f1f861..b9ed88b43 100644 --- a/src/queue/IQueueEnvironment.ts +++ b/src/queue/IQueueEnvironment.ts @@ -11,4 +11,5 @@ export default interface IQueueEnvironment { pwd(): string | undefined; debug(): Promise; inMemoryPersistence(): boolean; + extentMemoryLimit(): number | undefined; } diff --git a/src/queue/QueueEnvironment.ts b/src/queue/QueueEnvironment.ts index ec7c22a32..2b8c881f2 100644 --- a/src/queue/QueueEnvironment.ts +++ b/src/queue/QueueEnvironment.ts @@ -43,6 +43,10 @@ args ["", "inMemoryPersistence"], "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." ) + .option( + ["", "extentMemoryLimit"], + "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", + ) .option( ["d", "debug"], "Optional. Enable debug log by providing a valid local file path as log destination" @@ -119,6 +123,10 @@ export default class QueueEnvironment implements IQueueEnvironment { return false; } + public extentMemoryLimit(): number | undefined { + return this.flags.extentMemoryLimit; + } + public async debug(): Promise { if (typeof this.flags.debug === "string") { // Enable debug log to file diff --git a/src/queue/QueueServer.ts b/src/queue/QueueServer.ts index 385799d69..3910cfa7e 100644 --- a/src/queue/QueueServer.ts +++ b/src/queue/QueueServer.ts @@ -18,6 +18,7 @@ import IQueueMetadataStore from "./persistence/IQueueMetadataStore"; import LokiQueueMetadataStore from "./persistence/LokiQueueMetadataStore"; import QueueConfiguration from "./QueueConfiguration"; import QueueRequestListenerFactory from "./QueueRequestListenerFactory"; +import StorageError from "./errors/StorageError"; const BEFORE_CLOSE_MESSAGE = `Azurite Queue service is closing...`; const BEFORE_CLOSE_MESSAGE_GC_ERROR = `Azurite Queue service is closing... Critical error happens during GC.`; @@ -83,9 +84,11 @@ export default class QueueServer extends ServerBase { ); const extentStore: IExtentStore = configuration.isMemoryPersistence ? new MemoryExtentStore( + "queue", configuration.memoryStore ?? SharedChunkStore, extentMetadataStore, - logger + logger, + (sc, er, em, ri) => new StorageError(sc, er, em, ri) ) : new FSExtentStore( extentMetadataStore, configuration.persistencePathArray, diff --git a/src/queue/main.ts b/src/queue/main.ts index 8f22dabbb..ff0984390 100644 --- a/src/queue/main.ts +++ b/src/queue/main.ts @@ -12,6 +12,7 @@ import { DEFAULT_QUEUE_PERSISTENCE_ARRAY, DEFAULT_QUEUE_PERSISTENCE_PATH } from "./utils/constants"; +import { setExtentMemoryLimit } from "../common/ConfigurationBase"; // tslint:disable:no-console @@ -77,6 +78,8 @@ async function main() { // Create server instance const server = new QueueServer(config); + setExtentMemoryLimit(env); + // Start server console.log( `Azurite Queue service is starting on ${config.host}:${config.port}` diff --git a/src/table/main.ts b/src/table/main.ts index 63a760a75..29d5bd57f 100644 --- a/src/table/main.ts +++ b/src/table/main.ts @@ -29,7 +29,7 @@ async function main() { await access(dirname(debugFilePath)); } - // Store table configuation + // Store table configuration const config = new TableConfiguration( env.tableHost(), env.tablePort(), @@ -78,7 +78,7 @@ async function main() { }) .once("SIGINT", () => { console.log(beforeCloseMessage); - server.clean().then(() => { + server.close().then(() => { console.log(afterCloseMessage); }); }); diff --git a/tests/blob/memoryStore.unit.test.ts b/tests/blob/memoryStore.unit.test.ts index 185d8018c..db4b976f1 100644 --- a/tests/blob/memoryStore.unit.test.ts +++ b/tests/blob/memoryStore.unit.test.ts @@ -13,21 +13,19 @@ function chunk(id: string, count: number, fill?: string): IMemoryExtentChunk { describe("MemoryExtentChunkStore", () => { - it("should limit max size should work", () => { + it("should limit max size with try set", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 1000)) + store.set("blob", chunk("a", 1000)) - assert.throws( - () => store.set(chunk("b", 1)), - /Cannot add an extent chunk to the in-memory store. Size limit of 1000 bytes will be exceeded./) + assert.strictEqual(false, store.trySet("blob", chunk("b", 1))) }); it("updates current size for add", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 555)) - store.set(chunk("b", 1)) - store.set(chunk("c", 123)) + store.set("blob", chunk("a", 555)) + store.set("blob", chunk("b", 1)) + store.set("blob", chunk("c", 123)) assert.strictEqual(679, store.totalSize()) }); @@ -35,9 +33,9 @@ describe("MemoryExtentChunkStore", () => { it("updates current size based on count property", () => { const store = new MemoryExtentChunkStore(1000); - store.set({ + store.set("blob", { id: "a", - chunks: [Buffer.alloc(10, 'a'), Buffer.alloc(20, 'b')], + chunks: [Buffer.alloc(10, "a"), Buffer.alloc(20, "b")], count: 15, // a lie, for testing offset: 0 }) @@ -47,29 +45,29 @@ describe("MemoryExtentChunkStore", () => { it("updates current size for delete", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 555)) - store.set(chunk("b", 1)) - store.set(chunk("c", 123)) + store.set("blob", chunk("a", 555)) + store.set("blob", chunk("b", 1)) + store.set("blob", chunk("c", 123)) - store.delete("b") + store.delete("blob", "b") assert.strictEqual(678, store.totalSize()) }); it("allows size limit to be updated", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 20)) + store.set("blob", chunk("a", 20)) store.setSizeLimit(50) assert.throws( - () => store.set(chunk("b", 31)), + () => store.set("blob", chunk("b", 31)), /Cannot add an extent chunk to the in-memory store. Size limit of 50 bytes will be exceeded./) }); it("prevents size limit from being set lower than the current size", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 20)) + store.set("blob", chunk("a", 20)) assert.strictEqual(false, store.setSizeLimit(19)) assert.strictEqual(1000, store.sizeLimit()) @@ -77,43 +75,90 @@ describe("MemoryExtentChunkStore", () => { it("updates current size with delta when ID is replaced", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 555)) - store.set(chunk("b", 1)) + store.set("blob", chunk("a", 555)) + store.set("blob", chunk("b", 1)) - store.set(chunk("a", 123)) + store.set("blob", chunk("a", 123)) assert.strictEqual(124, store.totalSize()) }); it("resets current size for clear", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 555)) - store.set(chunk("b", 1)) - store.set(chunk("c", 123)) + store.set("blob", chunk("a", 555)) + store.set("blob", chunk("b", 1)) + store.set("blob", chunk("c", 123)) - store.clear() + store.clear("blob") assert.strictEqual(0, store.totalSize()) }); it("replaces buffers if ID is existing", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 11, '0')) + store.set("blob", chunk("a", 11, "0")) - store.set(chunk("a", 12, '1')) + store.set("blob", chunk("a", 12, "1")) - const existing = store.get('a') + const existing = store.get("blob", "a") assert.strictEqual(1, existing?.chunks.length) - assert.deepStrictEqual(Buffer.alloc(12, '1'), existing?.chunks[0]) + assert.deepStrictEqual(Buffer.alloc(12, "1"), existing?.chunks[0]) + }); + + it("keeps categories separate for set and delete", () => { + const store = new MemoryExtentChunkStore(1000); + store.set("queue", chunk("a", 12, "0")) + store.set("blob", chunk("a", 11, "1")) + + store.delete("blob", "a") + + const existing = store.get("queue", "a") + assert.deepStrictEqual([Buffer.alloc(12, "0")], existing?.chunks) + assert.strictEqual(12, store.totalSize()) + }); + + it("only clears a single category at a time", () => { + const store = new MemoryExtentChunkStore(1000); + store.set("queue", chunk("a", 12, "0")) + store.set("blob", chunk("a", 11, "1")) + + store.clear("queue") + + assert.strictEqual(11, store.totalSize()) + }); + + it("can clear all categories clears a single category at a time", () => { + const store = new MemoryExtentChunkStore(1000); + store.set("queue", chunk("a", 12, "0")) + store.set("blob", chunk("a", 11, "1")) + + store.clear("queue") + store.clear("blob") + + assert.strictEqual(0, store.totalSize()) + assert.strictEqual(undefined, store.get("queue", "a")) + assert.strictEqual(undefined, store.get("blob", "a")) + }); + + + it("all categories contribute to the same limit", () => { + const store = new MemoryExtentChunkStore(1000); + store.set("queue", chunk("a", 50, "0")) + store.set("blob", chunk("a", 50, "1")) + + const success = store.trySet("queue", chunk("b", 925)) + + assert.strictEqual(false, success) + assert.strictEqual(100, store.totalSize()) }); it("allows deletion by ID", () => { const store = new MemoryExtentChunkStore(1000); - store.set(chunk("a", 11, '0')) + store.set("blob", chunk("a", 11, "0")) - store.delete("a") + store.delete("blob", "a") - const existing = store.get('a') + const existing = store.get("blob", "a") assert.strictEqual(undefined, existing) }); From afb98fd6795ff084cd40f056a5f34cae6a1f2ecb Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 25 Oct 2023 19:14:39 -0400 Subject: [PATCH 27/37] Update spec with comments and add note about slower memory release --- ChangeLog.md | 2 +- README.md | 11 ++-- docs/designs/2023-10-in-memory-persistence.md | 65 ++++++++++++++++--- 3 files changed, 64 insertions(+), 14 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index f9898b46d..ac0ff1dc2 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -6,7 +6,7 @@ General: -- Add `--inMemoryStorage` option and related configs to persist all data in-memory without disk persistence. (issue #2227) +- Add `--inMemoryPersistence` option and related configs to persist all data in-memory without disk persistence. (issue #2227) ## 2023.10 Version 3.27.0 diff --git a/README.md b/README.md index b97f34676..120727569 100644 --- a/README.md +++ b/README.md @@ -258,8 +258,6 @@ Above command will try to start Azurite image with configurations: `--disableProductStyleUrl` force parsing storage account name from request Uri path, instead of from request Uri host. -`--inMemoryPersistence` disable persisting any data to disk. If the Azurite process is terminated, all data is lost. - > If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. > In above sample, you need to use **double first forward slash** for location and debug path parameters to avoid a [known issue](https://stackoverflow.com/questions/48427366/docker-build-command-add-c-program-files-git-to-the-path-passed-as-build-argu) for Git on Windows. @@ -441,7 +439,7 @@ This behavior can be disabled using this option. This setting is rejected when t enabled. ```cmd ---inMemoryStorage +--inMemoryPersistence ``` By default, the in-memory extent store (for blob and queue content) is limited to 50% of the total memory on the host @@ -451,8 +449,8 @@ option but virtual memory may be used if the limit exceeds the amount of availab operating system. A high limit may eventually lead to out of memory errors or reduced performance. The queue and blob extent storage count towards the same limit. The `--extentMemoryLimit` setting is rejected when -`--inMemoryStorage` is not specified. LokiJS storage (blob and queue metadata and table metadata and content) do not -contribute to this limit and are unbounded which is the same as without the `--inMemoryStorage` option. +`--inMemoryPersistence` is not specified. LokiJS storage (blob and queue metadata and table metadata and content) do not +contribute to this limit and are unbounded which is the same as without the `--inMemoryPersistence` option. ```cmd --extentMemoryLimit @@ -464,6 +462,9 @@ Well-behaved storage SDKs and tools will not a retry on this failure and will re error is met, consider deleting some in-memory content (blobs or queues), raising the limit, or restarting the Azurite server thus resetting the storage completely. +Note that if many hundreds of megabytes of content (queue message or blob content) are stored in-memory, it can take +noticeably longer than usual for the process to terminate since all the consumed memory needs to be released. + ### Command Line Options Differences between Azurite V2 Azurite V3 supports SharedKey, Account Shared Access Signature (SAS), Service SAS, OAuth, and Public Container Access authentications, you can use any Azure Storage SDKs or tools like Storage Explorer to connect Azurite V3 with any authentication strategy. diff --git a/docs/designs/2023-10-in-memory-persistence.md b/docs/designs/2023-10-in-memory-persistence.md index 1565ce2ee..bb5c03bd1 100644 --- a/docs/designs/2023-10-in-memory-persistence.md +++ b/docs/designs/2023-10-in-memory-persistence.md @@ -65,8 +65,8 @@ they need the disk storage. For other users where ephemeral storage is acceptabl those problems. Example issues: - [Azure/Azurite#1687](https://github.com/Azure/Azurite/issues/1687), [Azure/Azurite#804 - (comment)](https://github.com/Azure/Azurite/issues/803#issuecomment-1244987098), - additional permissions may be - needed for disk persistence, depending on the configured path + (comment)](https://github.com/Azure/Azurite/issues/803#issuecomment-1244987098) - additional permissions may be needed + for disk persistence, depending on the configured path - [Azure/Azurite#1967](https://github.com/Azure/Azurite/issues/1967) - there are OS limits for file handles ## Explanation @@ -78,9 +78,16 @@ A ``--inMemoryPersistence`` command line option will be introduced which will do 1. Switch all LokiJS instances from the default `fs` (file system) persistence model to `memory`. 1. Use a `MemoryExtentStore` for blob and queue extent storage instead of `FSExtentStore`. -The SQL-based persistence model will support the ``--inMemoryPersistence`` option however this particular use-case is -already strange because some storage for Azurite is on a potentially remote SQL instance and other storage is on the -Azurite local file system. This would simply allow the extent storage to instead be on the Azurite local memory. +By default, the `MemoryExtentStore` will limit itself to 50% of the total physical memory on the machine as returned by +[`os.totalmem()`](https://nodejs.org/api/os.html#ostotalmem). This can be overridden using a new `--extentMemoryLimit +` option. If this option is specified without `--inMemoryPersistence`, the CLI will error out. The `os.freemem()` +will not be queried because this will lead to unpredictable behavior for the user where an extent write operation (blob +content write or enqueue message) will fail intermittently based on the free memory. It is better to simple us a defined +amount of memory per physical machine and potential exceed the free memory available, thus leveraging virtual memory +available to the node process. + +The SQL-based persistence model will not support the in-memory persistence options and will error out if both the +`AZURITE_DB` environment variable is set and the either of the in-memory persistence options are specified. Similar options will be added to the various related configuration types, configuration providers, and the VS Code options. @@ -114,6 +121,13 @@ This will allow extent write operations to be stored in-memory (a list of buffer the list of chunks on demand. The implementation will be simpler than the `FSExtentStore` in that all extent chunks will have their own extent ID instead of sometimes being appended to existing extent chunks. +To support the implementation of `MemoryExtentStore` and to allow a shared memory limit between blob and queue extents, +a `MemoryExtentChunkStore` will be added which is the actual store of the `IMemoryExtentChunk` map mentioned above. Both +the blob and queue instances of `MemoryExtentStore` will share an instance of `MemoryExtentChunkStore` which allows the +proper bookkeeping of total bytes used. The `MemoryExtentChunkStore` will operate on two keys, `category: string` and +`id: string`. The category will either be `"blob"` or `"queue"` (allowing clean operations to clean only blob extents or +only queue extents) and the ID being the extent GUID mentioned before. + All unit tests will be run on both the existing disk-based persistence model and the in-memory model. There is one exception which is the `13. should find both old and new guids (backward compatible) when using guid type, @loki` test case. This operates on a legacy LokiJS disk-based DB which is a scenario that is not applicable to an in-memory model @@ -122,17 +136,42 @@ since old legacy DBs cannot be used. The test cases will use the in-memory-based persistence when the `AZURITE_TEST_INMEMORYPERSISTENCE` environment variable is set to a truthy value. +When a write operation will exceed the default 50% total memory limit or the limit specified by `--extentMemoryLimit`, +an HTTP 409 error will be returned to the called allowing tools and SDKs to avoid retries and notify the end user of the +problem. The error response message will look something like this: + +```http +HTTP/1.1 409 Cannot add an extent chunk to the in-memory store. Size limit of 4096 bytes will be exceeded +Server: Azurite-Blob/3.27.0 +x-ms-error-code: MemoryExtentStoreAtSizeLimit +x-ms-request-id: 9a7967f2-578f-47a4-8006-5ede890f89ff +Date: Wed, 25 Oct 2023 23:07:43 GMT +Connection: keep-alive +Keep-Alive: timeout=5 +Transfer-Encoding: chunked +Content-Type: application/xml + + + + MemoryExtentStoreAtSizeLimit + Cannot add an extent chunk to the in-memory store. Size limit of 4096 bytes will be exceeded +RequestId:9a7967f2-578f-47a4-8006-5ede890f89ff +Time:2023-10-25T23:07:43.658Z + +``` + ## Drawbacks Because extents are no longer stored on disk, the limit on the amount of data Azurite can store in this mode is no longer bounded by the available disk space but is instead bounded by the amount of memory available to the node process. +To mitigate this risk, the `--extentMemoryLimit` is defaulted to 50% of total memory. As a side note, the limit on the memory available to the node process is only loosely related to the physical memory of the host machine. Windows, Linux, and macOS all support virtual memory which allows blocks of memory to be paged onto disk. -As you can see, a load test of the in-memory implementation allows uploads exceeding 64 GiB (which was the amount of -physical memory available on the test machine). There will be a noticeable performance drop when this happens but it +As you can see below, a load test of the in-memory implementation allows uploads exceeding 64 GiB (which was the amount +of physical memory available on the test machine). There will be a noticeable performance drop when this happens but it illustrates how the storage available to `--inMemoryPersistence` can be quite large. ![high memory](meta/resources/2023-10-in-memory-persistence/high-memory.png) @@ -146,7 +185,12 @@ disk-based persistence. ## Rationale and alternatives -N/A +The in-memory extent store will not compact, concatenate or otherwise manipulate the byte arrays (`Buffer` instances) +that are provided by write operations. For large blobs, this can lead to a large array of chunks (based on the buffering +behavior provided by the web application and transport layer). This decision was made to reduce the number of bytes +copied during and upload operation and to avoid the need for a huge amount of contiguous memory. If needed, we can +evaluate concatenating some or all of the buffered chunks for performance or memory reasons. + ## Prior Art @@ -169,3 +213,8 @@ individually in all of these places: The current proposal allows only two options: allow these places to use disk-persistence (when `--inMemoryPersistence` is not specified) or none of these places to use disk-persistence (when ``--inMemoryPersistence`` is specified). + +Another future possibility would be to limit the memory used by LokiJS, but this would require support from the LokiJS +library and would likely be much more complex in nature because measuring the memory consumption of a heterogenous +object structure that LokiJS allows is very different that simply measuring the total number of bytes used in extents as +currently designed with `--extentMemoryLimit`. \ No newline at end of file From 0daf51c637219b01988ba415ae544ef4b5acdb6b Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 25 Oct 2023 20:35:59 -0400 Subject: [PATCH 28/37] Reject --location along with --inMemoryPersistence --- src/blob/BlobEnvironment.ts | 14 ++++++++++++-- src/common/ConfigurationBase.ts | 6 +----- src/common/Environment.ts | 12 +++++++++++- src/queue/QueueEnvironment.ts | 14 ++++++++++++-- src/table/TableEnvironment.ts | 8 ++++++-- src/table/main.ts | 3 ++- 6 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/blob/BlobEnvironment.ts b/src/blob/BlobEnvironment.ts index 75122e66e..cd2b97f5f 100644 --- a/src/blob/BlobEnvironment.ts +++ b/src/blob/BlobEnvironment.ts @@ -23,7 +23,8 @@ if (!(args as any).config.name) { .option( ["l", "location"], "Optional. Use an existing folder as workspace path, default is current working directory", - process.cwd() + "", + s => s == "" ? undefined : s ) .option( ["s", "silent"], @@ -42,11 +43,13 @@ if (!(args as any).config.name) { .option(["", "key"], "Optional. Path to certificate key .pem file") .option( ["", "inMemoryPersistence"], - "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost" ) .option( ["", "extentMemoryLimit"], "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", + -1, + s => s == -1 ? undefined : parseInt(s) ) .option( ["d", "debug"], @@ -128,7 +131,14 @@ export default class BlobEnvironment implements IBlobEnvironment { public inMemoryPersistence(): boolean { if (this.flags.inMemoryPersistence !== undefined) { + if (this.flags.location) { + throw new RangeError(`The --inMemoryPersistence option is not supported when the --location option is set.`) + } return true; + } else { + if (this.extentMemoryLimit() !== undefined) { + throw new RangeError(`The --extentMemoryLimit option is only supported when the --inMemoryPersistence option is set.`) + } } return false; } diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index 3c2e209f2..4f4200bf1 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -31,11 +31,7 @@ export function setExtentMemoryLimit(env: IBlobEnvironment | IQueueEnvironment) console.log(`In-memory extent storage is enabled with a limit of ${display} (${limit} bytes, ${totalPct}% of total memory).`); SharedChunkStore.setSizeLimit(limit); } else { - console.log(`In-memory storage is enabled with no limit on memory used.`); - } - } else { - if (env.extentMemoryLimit() !== undefined) { - throw new Error(`The --extentMemoryLimit option is only supported when the --inMemoryPersistence option is set.`) + console.log(`In-memory extent storage is enabled with no limit on memory used.`); } } } diff --git a/src/common/Environment.ts b/src/common/Environment.ts index 49994fdb2..8e0968ee3 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -51,7 +51,8 @@ args .option( ["l", "location"], "Optional. Use an existing folder as workspace path, default is current working directory", - process.cwd() + "", + s => s == "" ? undefined : s ) .option(["s", "silent"], "Optional. Disable access log displayed in console") .option( @@ -77,6 +78,8 @@ args .option( ["", "extentMemoryLimit"], "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", + -1, + s => s == -1 ? undefined : parseInt(s) ) .option( ["d", "debug"], @@ -165,7 +168,14 @@ export default class Environment implements IEnvironment { public inMemoryPersistence(): boolean { if (this.flags.inMemoryPersistence !== undefined) { + if (this.flags.location) { + throw new RangeError(`The --inMemoryPersistence option is not supported when the --location option is set.`) + } return true; + } else { + if (this.extentMemoryLimit() !== undefined) { + throw new RangeError(`The --extentMemoryLimit option is only supported when the --inMemoryPersistence option is set.`) + } } return false; } diff --git a/src/queue/QueueEnvironment.ts b/src/queue/QueueEnvironment.ts index 2b8c881f2..54669fcd6 100644 --- a/src/queue/QueueEnvironment.ts +++ b/src/queue/QueueEnvironment.ts @@ -20,7 +20,8 @@ args .option( ["l", "location"], "Optional. Use an existing folder as workspace path, default is current working directory", - process.cwd() + "", + s => s == "" ? undefined : s ) .option(["s", "silent"], "Optional. Disable access log displayed in console") .option( @@ -41,11 +42,13 @@ args .option(["", "pwd"], "Optional. Password for .pfx file") .option( ["", "inMemoryPersistence"], - "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost" ) .option( ["", "extentMemoryLimit"], "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", + -1, + s => s == -1 ? undefined : parseInt(s) ) .option( ["d", "debug"], @@ -118,7 +121,14 @@ export default class QueueEnvironment implements IQueueEnvironment { public inMemoryPersistence(): boolean { if (this.flags.inMemoryPersistence !== undefined) { + if (this.flags.location) { + throw new RangeError(`The --inMemoryPersistence option is not supported when the --location option is set.`) + } return true; + } else { + if (this.extentMemoryLimit() !== undefined) { + throw new RangeError(`The --extentMemoryLimit option is only supported when the --inMemoryPersistence option is set.`) + } } return false; } diff --git a/src/table/TableEnvironment.ts b/src/table/TableEnvironment.ts index 0447b4eaf..bc4477a9e 100644 --- a/src/table/TableEnvironment.ts +++ b/src/table/TableEnvironment.ts @@ -23,7 +23,8 @@ args .option( ["l", "location"], "Optional. Use an existing folder as workspace path, default is current working directory", - process.cwd() + "", + s => s == "" ? undefined : s ) .option(["s", "silent"], "Optional. Disable access log displayed in console") .option( @@ -40,7 +41,7 @@ args ) .option( ["", "inMemoryPersistence"], - "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost." + "Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost" ) .option( ["d", "debug"], @@ -106,6 +107,9 @@ export default class TableEnvironment implements ITableEnvironment { public inMemoryPersistence(): boolean { if (this.flags.inMemoryPersistence !== undefined) { + if (this.flags.location) { + throw new RangeError(`The --inMemoryPersistence option is not supported when the --location option is set.`) + } return true; } return false; diff --git a/src/table/main.ts b/src/table/main.ts index 29d5bd57f..7f497cd14 100644 --- a/src/table/main.ts +++ b/src/table/main.ts @@ -44,7 +44,8 @@ async function main() { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl() + env.disableProductStyleUrl(), + env.inMemoryPersistence(), ); // We use logger singleton as global debugger logger to track detailed outputs cross layers From 97acd4e58e86cced7e14e0c2b3655af5b97287f1 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 25 Oct 2023 20:46:54 -0400 Subject: [PATCH 29/37] Fail fast on the binary tests if the binaries don't exist --- tests/exe.test.ts | 23 ++++++++++++++++++++--- tests/linuxbinary.test.ts | 23 ++++++++++++++++++++--- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/tests/exe.test.ts b/tests/exe.test.ts index e1e70387b..73b67cecf 100644 --- a/tests/exe.test.ts +++ b/tests/exe.test.ts @@ -36,6 +36,7 @@ import { overrideRequest, restoreBuildRequestOptions } from "./testutils"; +import { existsSync } from "fs"; // server address used for testing. Note that Azurite.exe has // server address of http://127.0.0.1:10000 and so on by default @@ -53,6 +54,14 @@ configLogger(false); // Azure Storage Connection String (using SAS or Key). const testLocalAzuriteInstance = true; +const binaryPath = ".\\release\\azurite.exe" + +function throwOnMissingBinary() { + if (!existsSync(binaryPath)) { + throw new Error("The Windows binary does not exist. You must build it first using 'npm run build:exe'.") + } +} + describe("exe test", () => { const tableService = Azure.createTableService( createConnectionStringForTest(testLocalAzuriteInstance) @@ -65,11 +74,15 @@ describe("exe test", () => { let childPid: number; + beforeEach(() => throwOnMissingBinary()) + before(async () => { + throwOnMissingBinary() + overrideRequest(requestOverride, tableService); tableName = getUniqueName("table"); const child = execFile( - ".\\release\\azurite.exe", + binaryPath, ["--blobPort 11000", "--queuePort 11001", "--tablePort 11002"], { cwd: process.cwd(), shell: true, env: {} } ); @@ -116,10 +129,14 @@ describe("exe test", () => { tableService.removeAllListeners(); await find("name", "azurite.exe", true).then((list: any) => { - process.kill(list[0].pid); + if (list.length > 0) { + process.kill(list[0].pid); + } }); - process.kill(childPid); + if (childPid) { + process.kill(childPid); + } }); describe("table test", () => { diff --git a/tests/linuxbinary.test.ts b/tests/linuxbinary.test.ts index 4a888f27a..7bfcac2ef 100644 --- a/tests/linuxbinary.test.ts +++ b/tests/linuxbinary.test.ts @@ -23,6 +23,7 @@ import { bodyToString, EMULATOR_ACCOUNT_KEY, EMULATOR_ACCOUNT_NAME, getUniqueName, overrideRequest, restoreBuildRequestOptions } from './testutils'; +import { existsSync } from 'fs'; // server address used for testing. Note that Azuritelinux has // server address of http://127.0.0.1:10000 and so on by default @@ -40,6 +41,14 @@ configLogger(false); // Azure Storage Connection String (using SAS or Key). const testLocalAzuriteInstance = true; +const binaryPath = "./release/azuritelinux" + +function throwOnMissingBinary() { + if (!existsSync(binaryPath)) { + throw new Error("The Linux binary does not exist. You must build it first using 'npm run build:linux'.") + } +} + describe("linux binary test", () => { const tableService = Azure.createTableService( @@ -53,10 +62,14 @@ describe("linux binary test", () => { let childPid: number; + beforeEach(() => throwOnMissingBinary()) + before(async () => { + throwOnMissingBinary() + overrideRequest(requestOverride, tableService); tableName = getUniqueName("table"); - const child = execFile("./release/azuritelinux", ["--blobPort 11000", "--queuePort 11001", "--tablePort 11002"], { cwd: process.cwd(), shell: true, env: {} }); + const child = execFile(binaryPath, ["--blobPort 11000", "--queuePort 11001", "--tablePort 11002"], { cwd: process.cwd(), shell: true, env: {} }); childPid = child.pid; @@ -89,10 +102,14 @@ describe("linux binary test", () => { tableService.removeAllListeners(); await find('name', 'azuritelinux', true).then((list: any) => { - process.kill(list[0].pid); + if (list.length > 0) { + process.kill(list[0].pid); + } }); - process.kill(childPid); + if (childPid) { + process.kill(childPid); + } }); describe("table test", () => { From 68938c8dbfb8ebcd5f1b843b655f2988c4b08586 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 25 Oct 2023 21:06:33 -0400 Subject: [PATCH 30/37] Properly tag tests --- tests/blob/memoryStore.unit.test.ts | 35 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/tests/blob/memoryStore.unit.test.ts b/tests/blob/memoryStore.unit.test.ts index db4b976f1..1378d38dc 100644 --- a/tests/blob/memoryStore.unit.test.ts +++ b/tests/blob/memoryStore.unit.test.ts @@ -13,14 +13,14 @@ function chunk(id: string, count: number, fill?: string): IMemoryExtentChunk { describe("MemoryExtentChunkStore", () => { - it("should limit max size with try set", () => { + it("should limit max size with try set @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 1000)) assert.strictEqual(false, store.trySet("blob", chunk("b", 1))) }); - it("updates current size for add", () => { + it("updates current size for add @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 555)) @@ -30,7 +30,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(679, store.totalSize()) }); - it("updates current size based on count property", () => { + it("updates current size based on count property @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", { @@ -43,7 +43,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(15, store.totalSize()) }); - it("updates current size for delete", () => { + it("updates current size for delete @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 555)) store.set("blob", chunk("b", 1)) @@ -54,7 +54,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(678, store.totalSize()) }); - it("allows size limit to be updated", () => { + it("allows size limit to be updated @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 20)) @@ -65,7 +65,7 @@ describe("MemoryExtentChunkStore", () => { /Cannot add an extent chunk to the in-memory store. Size limit of 50 bytes will be exceeded./) }); - it("prevents size limit from being set lower than the current size", () => { + it("prevents size limit from being set lower than the current size @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 20)) @@ -73,7 +73,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(1000, store.sizeLimit()) }); - it("updates current size with delta when ID is replaced", () => { + it("updates current size with delta when ID is replaced @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 555)) store.set("blob", chunk("b", 1)) @@ -83,7 +83,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(124, store.totalSize()) }); - it("resets current size for clear", () => { + it("resets current size for clear @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 555)) store.set("blob", chunk("b", 1)) @@ -94,7 +94,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(0, store.totalSize()) }); - it("replaces buffers if ID is existing", () => { + it("replaces buffers if ID is existing @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 11, "0")) @@ -105,7 +105,7 @@ describe("MemoryExtentChunkStore", () => { assert.deepStrictEqual(Buffer.alloc(12, "1"), existing?.chunks[0]) }); - it("keeps categories separate for set and delete", () => { + it("keeps categories separate for set and delete @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("queue", chunk("a", 12, "0")) store.set("blob", chunk("a", 11, "1")) @@ -117,7 +117,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(12, store.totalSize()) }); - it("only clears a single category at a time", () => { + it("only clears a single category at a time @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("queue", chunk("a", 12, "0")) store.set("blob", chunk("a", 11, "1")) @@ -127,7 +127,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(11, store.totalSize()) }); - it("can clear all categories clears a single category at a time", () => { + it("can clear all categories clears a single category at a time @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("queue", chunk("a", 12, "0")) store.set("blob", chunk("a", 11, "1")) @@ -140,8 +140,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(undefined, store.get("blob", "a")) }); - - it("all categories contribute to the same limit", () => { + it("all categories contribute to the same limit @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("queue", chunk("a", 50, "0")) store.set("blob", chunk("a", 50, "1")) @@ -152,7 +151,7 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(100, store.totalSize()) }); - it("allows deletion by ID", () => { + it("allows deletion by ID @loki", () => { const store = new MemoryExtentChunkStore(1000); store.set("blob", chunk("a", 11, "0")) @@ -162,9 +161,9 @@ describe("MemoryExtentChunkStore", () => { assert.strictEqual(undefined, existing) }); - it("should have a shared instance defaulting to close to 50% of the total bytes", () => { + it("should have a shared instance defaulting to close to 50% of the total bytes @loki", () => { assert.ok(SharedChunkStore.sizeLimit(), "The default store's size limit should be set.") - assert.ok(SharedChunkStore.sizeLimit()! > 0.4 * totalmem()) - assert.ok(SharedChunkStore.sizeLimit()! < 0.6 * totalmem()) + assert.ok(SharedChunkStore.sizeLimit()! > 0.49 * totalmem()) + assert.ok(SharedChunkStore.sizeLimit()! < 0.51 * totalmem()) }); }); From 69286024aca004853d5601f472ce2d164b5c7541 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Thu, 26 Oct 2023 09:28:45 -0400 Subject: [PATCH 31/37] Polish READMEs --- README.mcr.md | 2 -- README.md | 28 +++++++++++++++++----------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/README.mcr.md b/README.mcr.md index f9bd7fc8c..531c4cfd1 100644 --- a/README.mcr.md +++ b/README.mcr.md @@ -74,8 +74,6 @@ Above command will try to start Azurite image with configurations: `--disableProductStyleUrl` force parsing storage account name from request Uri path, instead of from request Uri host. -`--inMemoryPersistence` disable persisting any data to disk. If the Azurite process is terminated, all data is lost. - > If you use customized azurite paramters for docker image, `--blobHost 0.0.0.0`, `--queueHost 0.0.0.0` are required parameters. > In above sample, you need to use **double first forward slash** for location and debug path parameters to avoid a [known issue](https://stackoverflow.com/questions/48427366/docker-build-command-add-c-program-files-git-to-the-path-passed-as-build-argu) for Git on Windows. diff --git a/README.md b/README.md index 120727569..c8f5af40d 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,15 @@ - [Azurite V3](#azurite-v3) - [Introduction](#introduction) - - [Features & Key Changes in Azurite V3](#features--key-changes-in-azurite-v3) + - [Features \& Key Changes in Azurite V3](#features--key-changes-in-azurite-v3) - [Getting Started](#getting-started) - [GitHub](#github) - [NPM](#npm) - [Visual Studio Code Extension](#visual-studio-code-extension) - [DockerHub](#dockerhub) + - [Run Azurite V3 docker image](#run-azurite-v3-docker-image) + - [Run Azurite V3 docker image with customized persisted data location](#run-azurite-v3-docker-image-with-customized-persisted-data-location) + - [Customize all Azurite V3 supported parameters for docker image](#customize-all-azurite-v3-supported-parameters-for-docker-image) - [Docker Compose](#docker-compose) - [NuGet](#nuget) - [Visual Studio](#visual-studio) @@ -33,24 +36,26 @@ - [Certificate Configuration (HTTPS)](#certificate-configuration-https) - [OAuth Configuration](#oauth-configuration) - [Skip API Version Check](#skip-api-version-check) + - [Disable Product Style Url](#disable-product-style-url) + - [Use in-memory storage](#use-in-memory-storage) - [Command Line Options Differences between Azurite V2](#command-line-options-differences-between-azurite-v2) - [Supported Environment Variable Options](#supported-environment-variable-options) - - [Customized Storage Accounts & Keys](#customized-storage-accounts--keys) + - [Customized Storage Accounts \& Keys](#customized-storage-accounts--keys) - [Customized Metadata Storage by External Database (Preview)](#customized-metadata-storage-by-external-database-preview) - [HTTPS Setup](#https-setup) - [PEM](#pem) - [PFX](#pfx) - [Usage with Azure Storage SDKs or Tools](#usage-with-azure-storage-sdks-or-tools) - [Default Storage Account](#default-storage-account) - - [Customized Storage Accounts & Keys](#customized-storage-accounts--keys-1) + - [Customized Storage Accounts \& Keys](#customized-storage-accounts--keys-1) - [Connection Strings](#connection-strings) - [Azure SDKs](#azure-sdks) - [Storage Explorer](#storage-explorer) - [Workspace Structure](#workspace-structure) - [Differences between Azurite and Azure Storage](#differences-between-azurite-and-azure-storage) - [Storage Accounts](#storage-accounts) - - [Endpoint & Connection URL](#endpoint--connection-url) - - [Scalability & Performance](#scalability--performance) + - [Endpoint \& Connection URL](#endpoint--connection-url) + - [Scalability \& Performance](#scalability--performance) - [Error Handling](#error-handling) - [API Version Compatible Strategy](#api-version-compatible-strategy) - [RA-GRS](#ra-grs) @@ -199,6 +204,7 @@ Following extension configurations are supported: - `azurite.skipApiVersionCheck` Skip the request API version check, by default false. - `azurite.disableProductStyleUrl` Force parsing storage account name from request Uri path, instead of from request Uri host. - `azurite.inMemoryPersistence` Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. +- `azurite.memoryExtentLimit` When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of bytes. Defaults to 50% of total memory. ### [DockerHub](https://hub.docker.com/_/microsoft-azure-storage-azurite) @@ -433,10 +439,10 @@ Optional. When using FQDN instead of IP in request Uri host, by default Azurite ### Use in-memory storage -Optional. Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. By default, -LokiJS persists blob and queue metadata to disk and content to extent files. Table storage persists all data to disk. -This behavior can be disabled using this option. This setting is rejected when the SQL based metadata implementation is -enabled. +Optional. Disable persisting any data to disk and only store data in-memory. If the Azurite process is terminated, all +data is lost. By default, LokiJS persists blob and queue metadata to disk and content to extent files. Table storage +persists all data to disk. This behavior can be disabled using this option. This setting is rejected when the SQL based +metadata implementation is enabled. ```cmd --inMemoryPersistence @@ -449,8 +455,8 @@ option but virtual memory may be used if the limit exceeds the amount of availab operating system. A high limit may eventually lead to out of memory errors or reduced performance. The queue and blob extent storage count towards the same limit. The `--extentMemoryLimit` setting is rejected when -`--inMemoryPersistence` is not specified. LokiJS storage (blob and queue metadata and table metadata and content) do not -contribute to this limit and are unbounded which is the same as without the `--inMemoryPersistence` option. +`--inMemoryPersistence` is not specified. LokiJS storage (blob and queue metadata and table metadata and content) does +not contribute to this limit and is unbounded which is the same as without the `--inMemoryPersistence` option. ```cmd --extentMemoryLimit From 8d43f65e56ec0fecaef9304b90e3b062fbdf3311 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sat, 4 Nov 2023 09:51:06 -0400 Subject: [PATCH 32/37] Address code review comments --- ChangeLog.md | 2 +- README.md | 11 +++++------ tests/blob/blockblob.highlevel.test.ts | 10 +++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index ac0ff1dc2..b6c364f56 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -6,7 +6,7 @@ General: -- Add `--inMemoryPersistence` option and related configs to persist all data in-memory without disk persistence. (issue #2227) +- Add `--inMemoryPersistence` and `--extentMemoryLimit` options and related configs to store all data in-memory without disk persistence. (issue #2227) ## 2023.10 Version 3.27.0 diff --git a/README.md b/README.md index c8f5af40d..1e4f1d71d 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,6 @@ - [NPM](#npm) - [Visual Studio Code Extension](#visual-studio-code-extension) - [DockerHub](#dockerhub) - - [Run Azurite V3 docker image](#run-azurite-v3-docker-image) - - [Run Azurite V3 docker image with customized persisted data location](#run-azurite-v3-docker-image-with-customized-persisted-data-location) - - [Customize all Azurite V3 supported parameters for docker image](#customize-all-azurite-v3-supported-parameters-for-docker-image) - - [Docker Compose](#docker-compose) - [NuGet](#nuget) - [Visual Studio](#visual-studio) - [Supported Command Line Options](#supported-command-line-options) @@ -204,7 +200,7 @@ Following extension configurations are supported: - `azurite.skipApiVersionCheck` Skip the request API version check, by default false. - `azurite.disableProductStyleUrl` Force parsing storage account name from request Uri path, instead of from request Uri host. - `azurite.inMemoryPersistence` Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. -- `azurite.memoryExtentLimit` When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of bytes. Defaults to 50% of total memory. +- `azurite.extentMemoryLimit` When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of bytes. This does not limit blob, queue, or table metadata. Defaults to 50% of total memory. ### [DockerHub](https://hub.docker.com/_/microsoft-azure-storage-azurite) @@ -442,7 +438,8 @@ Optional. When using FQDN instead of IP in request Uri host, by default Azurite Optional. Disable persisting any data to disk and only store data in-memory. If the Azurite process is terminated, all data is lost. By default, LokiJS persists blob and queue metadata to disk and content to extent files. Table storage persists all data to disk. This behavior can be disabled using this option. This setting is rejected when the SQL based -metadata implementation is enabled. +metadata implementation is enabled (via `AZURITE_DB`). This setting is rejected when the `--location` option is +specified. ```cmd --inMemoryPersistence @@ -462,6 +459,8 @@ not contribute to this limit and is unbounded which is the same as without the ` --extentMemoryLimit ``` +This option is rejected when `--inMemoryPersistence` is not specified. + When the limit is reached, write operations to the blob or queue endpoints which carry content will fail with an `HTTP 409` status code, a custom storage error code of `MemoryExtentStoreAtSizeLimit`, and a helpful error message. Well-behaved storage SDKs and tools will not a retry on this failure and will return a related error message. If this diff --git a/tests/blob/blockblob.highlevel.test.ts b/tests/blob/blockblob.highlevel.test.ts index 999b8df68..4c5f0cf05 100644 --- a/tests/blob/blockblob.highlevel.test.ts +++ b/tests/blob/blockblob.highlevel.test.ts @@ -179,7 +179,7 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) {} + } catch (err) { } assert.ok(eventTriggered); }).timeout(timeoutForLargeFileUploadingTest); @@ -198,7 +198,7 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) {} + } catch (err) { } assert.ok(eventTriggered); }); @@ -260,7 +260,7 @@ describe("BlockBlobHighlevel", () => { abortSignal: AbortController.timeout(1) }); assert.fail(); - } catch (err:any) { + } catch (err: any) { assert.ok((err.message as string).toLowerCase().includes("abort")); } }).timeout(timeoutForLargeFileUploadingTest); @@ -306,7 +306,7 @@ describe("BlockBlobHighlevel", () => { const aborter = new AbortController(); try { await blockBlobClient.downloadToBuffer(buf, 0, undefined, { - blockSize: 32 * 1024, // if too small, the test is very slow + blockSize: 1 * 1024, maxRetryRequestsPerBlock: 5, concurrency: 1, onProgress: () => { @@ -314,7 +314,7 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) {} + } catch (err) { } assert.ok(eventTriggered); }).timeout(timeoutForLargeFileUploadingTest); From 12dee7254c14cb1a2a5a368706516ff34cd00073 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sat, 4 Nov 2023 09:58:03 -0400 Subject: [PATCH 33/37] Eliminate unnecessary diff noise --- README.md | 11 ++++++----- azure-pipelines.yml | 8 ++++---- src/blob/BlobConfiguration.ts | 2 +- src/blob/BlobServerFactory.ts | 2 +- src/blob/SqlBlobConfiguration.ts | 4 ++-- src/common/ConfigurationBase.ts | 7 ++++--- src/queue/QueueConfiguration.ts | 2 +- src/queue/gc/QueueGCManager.ts | 8 ++++---- src/queue/persistence/LokiQueueMetadataStore.ts | 8 ++++---- src/table/TableConfiguration.ts | 2 +- tests/blob/blockblob.highlevel.test.ts | 8 ++++---- tests/table/apis/table.validation.rest.test.ts | 10 +++++----- 12 files changed, 37 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 1e4f1d71d..fb80ae146 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,13 @@ - [Azurite V3](#azurite-v3) - [Introduction](#introduction) - - [Features \& Key Changes in Azurite V3](#features--key-changes-in-azurite-v3) + - [Features & Key Changes in Azurite V3](#features--key-changes-in-azurite-v3) - [Getting Started](#getting-started) - [GitHub](#github) - [NPM](#npm) - [Visual Studio Code Extension](#visual-studio-code-extension) - [DockerHub](#dockerhub) + - [Docker Compose](#docker-compose) - [NuGet](#nuget) - [Visual Studio](#visual-studio) - [Supported Command Line Options](#supported-command-line-options) @@ -36,22 +37,22 @@ - [Use in-memory storage](#use-in-memory-storage) - [Command Line Options Differences between Azurite V2](#command-line-options-differences-between-azurite-v2) - [Supported Environment Variable Options](#supported-environment-variable-options) - - [Customized Storage Accounts \& Keys](#customized-storage-accounts--keys) + - [Customized Storage Accounts & Keys](#customized-storage-accounts--keys) - [Customized Metadata Storage by External Database (Preview)](#customized-metadata-storage-by-external-database-preview) - [HTTPS Setup](#https-setup) - [PEM](#pem) - [PFX](#pfx) - [Usage with Azure Storage SDKs or Tools](#usage-with-azure-storage-sdks-or-tools) - [Default Storage Account](#default-storage-account) - - [Customized Storage Accounts \& Keys](#customized-storage-accounts--keys-1) + - [Customized Storage Accounts & Keys](#customized-storage-accounts--keys-1) - [Connection Strings](#connection-strings) - [Azure SDKs](#azure-sdks) - [Storage Explorer](#storage-explorer) - [Workspace Structure](#workspace-structure) - [Differences between Azurite and Azure Storage](#differences-between-azurite-and-azure-storage) - [Storage Accounts](#storage-accounts) - - [Endpoint \& Connection URL](#endpoint--connection-url) - - [Scalability \& Performance](#scalability--performance) + - [Endpoint & Connection URL](#endpoint--connection-url) + - [Scalability & Performance](#scalability--performance) - [Error Handling](#error-handling) - [API Version Compatible Strategy](#api-version-compatible-strategy) - [RA-GRS](#ra-grs) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 598f86734..92fdcc4ad 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,7 +37,7 @@ jobs: workingDirectory: "./" displayName: "npm run test:blob" env: {} - + - script: | npm run test:blob:in-memory workingDirectory: "./" @@ -59,12 +59,12 @@ jobs: inputs: versionSpec: "$(node_version)" displayName: "Install Node.js" - + - script: | npm ci --legacy-peer-deps workingDirectory: "./" displayName: "npm ci --legacy-peer-deps" - + - script: | npm run test:blob workingDirectory: "./" @@ -386,7 +386,7 @@ jobs: strategy: matrix: # Table tests no longer suport older node versions - # skip node 14 Azurite install test, since it has issue iwth new npm, which is not azurite issue. + # skip node 14 Azurite install test, since it has issue iwth new npm, which is not azurite issue. # Track with https://github.com/Azure/Azurite/issues/1550. Will add node 14 back later when the issue resolved. #node_14_x: # node_version: 14.x diff --git a/src/blob/BlobConfiguration.ts b/src/blob/BlobConfiguration.ts index cc31fa5a9..45733c0f9 100644 --- a/src/blob/BlobConfiguration.ts +++ b/src/blob/BlobConfiguration.ts @@ -57,7 +57,7 @@ export default class BlobConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl, + disableProductStyleUrl ); } } diff --git a/src/blob/BlobServerFactory.ts b/src/blob/BlobServerFactory.ts index 1ea4f8455..0359e5242 100644 --- a/src/blob/BlobServerFactory.ts +++ b/src/blob/BlobServerFactory.ts @@ -66,7 +66,7 @@ export class BlobServerFactory { env.key(), env.pwd(), env.oauth(), - env.disableProductStyleUrl(), + env.disableProductStyleUrl() ); return new SqlBlobServer(config); diff --git a/src/blob/SqlBlobConfiguration.ts b/src/blob/SqlBlobConfiguration.ts index 093175a7b..a6e8b0586 100644 --- a/src/blob/SqlBlobConfiguration.ts +++ b/src/blob/SqlBlobConfiguration.ts @@ -35,7 +35,7 @@ export default class SqlBlobConfiguration extends ConfigurationBase { key: string = "", pwd: string = "", oauth?: string, - disableProductStyleUrl: boolean = false, + disableProductStyleUrl: boolean = false ) { super( host, @@ -50,7 +50,7 @@ export default class SqlBlobConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl, + disableProductStyleUrl ); } } diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index 4f4200bf1..37a793480 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -51,7 +51,7 @@ export default abstract class ConfigurationBase { public readonly pwd: string = "", public readonly oauth?: string, public readonly disableProductStyleUrl: boolean = false, - ) { } + ) {} public hasCert() { if (this.cert.length > 0 && this.key.length > 0) { @@ -92,7 +92,8 @@ export default abstract class ConfigurationBase { } public getHttpServerAddress(): string { - return `http${this.hasCert() === CertOptions.Default ? "" : "s"}://${this.host - }:${this.port}`; + return `http${this.hasCert() === CertOptions.Default ? "" : "s"}://${ + this.host + }:${this.port}`; } } diff --git a/src/queue/QueueConfiguration.ts b/src/queue/QueueConfiguration.ts index b98be78c9..31f52257c 100644 --- a/src/queue/QueueConfiguration.ts +++ b/src/queue/QueueConfiguration.ts @@ -57,7 +57,7 @@ export default class QueueConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl, + disableProductStyleUrl ); } } diff --git a/src/queue/gc/QueueGCManager.ts b/src/queue/gc/QueueGCManager.ts index ec58a1726..d916c8756 100644 --- a/src/queue/gc/QueueGCManager.ts +++ b/src/queue/gc/QueueGCManager.ts @@ -145,18 +145,18 @@ export default class QueueGCManager implements IGCManager { private async markSweepLoop(): Promise { while (this._status === Status.Running) { this.logger.info( - `QueueGCManager:markSweepLoop() Start next mark and sweep.` + `QueueGCManager:markSweepLoop() Start new mark and sweep.` ); const start = Date.now(); await this.markSweep(); - const period = Date.now() - start; + const duration = Date.now() - start; this.logger.info( - `QueueGCManager:markSweepLoop() Mark and sweep finished, taken ${period}ms.` + `QueueGCManager:markSweepLoop() Mark and sweep finished, take ${duration}ms.` ); if (this._status === Status.Running) { this.logger.info( - `QueueGCManager:markSweepLoop() Sleep for ${this.gcIntervalInMS}ms.` + `QueueGCManager:markSweepLoop() Sleep for ${this.gcIntervalInMS}` ); await this.sleep(this.gcIntervalInMS); } diff --git a/src/queue/persistence/LokiQueueMetadataStore.ts b/src/queue/persistence/LokiQueueMetadataStore.ts index 0cd285172..56b376647 100644 --- a/src/queue/persistence/LokiQueueMetadataStore.ts +++ b/src/queue/persistence/LokiQueueMetadataStore.ts @@ -236,10 +236,10 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore { prefix === "" ? { $loki: { $gt: marker }, accountName: account } : { - name: { $regex: `^${this.escapeRegex(prefix)}` }, - $loki: { $gt: marker }, - accountName: account - }; + name: { $regex: `^${this.escapeRegex(prefix)}` }, + $loki: { $gt: marker }, + accountName: account + }; // Get one more item to help check if the query reach the tail of the collection. const docs = coll diff --git a/src/table/TableConfiguration.ts b/src/table/TableConfiguration.ts index 3fe7308bf..7b7e89717 100644 --- a/src/table/TableConfiguration.ts +++ b/src/table/TableConfiguration.ts @@ -51,7 +51,7 @@ export default class TableConfiguration extends ConfigurationBase { key, pwd, oauth, - disableProductStyleUrl, + disableProductStyleUrl ); } } diff --git a/tests/blob/blockblob.highlevel.test.ts b/tests/blob/blockblob.highlevel.test.ts index 4c5f0cf05..f2057eeed 100644 --- a/tests/blob/blockblob.highlevel.test.ts +++ b/tests/blob/blockblob.highlevel.test.ts @@ -179,7 +179,7 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) { } + } catch (err) {} assert.ok(eventTriggered); }).timeout(timeoutForLargeFileUploadingTest); @@ -198,7 +198,7 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) { } + } catch (err) {} assert.ok(eventTriggered); }); @@ -260,7 +260,7 @@ describe("BlockBlobHighlevel", () => { abortSignal: AbortController.timeout(1) }); assert.fail(); - } catch (err: any) { + } catch (err:any) { assert.ok((err.message as string).toLowerCase().includes("abort")); } }).timeout(timeoutForLargeFileUploadingTest); @@ -314,7 +314,7 @@ describe("BlockBlobHighlevel", () => { aborter.abort(); } }); - } catch (err) { } + } catch (err) {} assert.ok(eventTriggered); }).timeout(timeoutForLargeFileUploadingTest); diff --git a/tests/table/apis/table.validation.rest.test.ts b/tests/table/apis/table.validation.rest.test.ts index 524339a4e..a891d5bbc 100644 --- a/tests/table/apis/table.validation.rest.test.ts +++ b/tests/table/apis/table.validation.rest.test.ts @@ -6,7 +6,7 @@ import * as assert from "assert"; import { configLogger } from "../../../src/common/Logger"; import TableServer from "../../../src/table/TableServer"; -import { getUniqueName } from "../../testutils"; +import { getUniqueName } from "../../testutils"; import { deleteToAzurite, getToAzurite, @@ -401,7 +401,7 @@ describe("table name validation tests", () => { } }); - + it(`Should work with production style URL when ${productionStyleHostName} is resolvable`, async () => { await dns.promises.lookup(productionStyleHostName).then( async (lookupAddress) => { @@ -414,7 +414,7 @@ describe("table name validation tests", () => { Accept: "application/json;odata=nometadata" }; try { - let response = await postToAzuriteProductionUrl(productionStyleHostName, "Tables", body, createTableHeaders); + let response = await postToAzuriteProductionUrl(productionStyleHostName,"Tables", body, createTableHeaders); assert.strictEqual(response.status, 201); } catch (err: any) { assert.fail(); @@ -446,9 +446,9 @@ describe("table name validation tests", () => { Accept: "application/json;odata=nometadata" }; try { - let response = await postToAzuriteProductionUrl(productionStyleHostName, "Tables", body, createTableHeaders); + let response = await postToAzuriteProductionUrl(productionStyleHostName,"Tables", body, createTableHeaders); assert.strictEqual(response.status, 201); - let tablesList = await getToAzuriteProductionUrl(productionStyleHostNameForSecondary, "Tables", createTableHeaders); + let tablesList = await getToAzuriteProductionUrl(productionStyleHostNameForSecondary,"Tables", createTableHeaders); assert.strictEqual(tablesList.status, 200); } catch (err: any) { assert.fail(); From 17906ab7cc1b5cec4b435f48693d6aa833f99168 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Sun, 5 Nov 2023 12:23:45 -0500 Subject: [PATCH 34/37] Change extentMemoryLimit to use megabytes as the unit --- README.md | 8 ++--- docs/designs/2023-10-in-memory-persistence.md | 2 +- package.json | 2 +- src/blob/BlobEnvironment.ts | 4 +-- src/common/ConfigurationBase.ts | 30 ++++++------------- src/common/Environment.ts | 4 +-- src/common/VSCServerManagerClosedState.ts | 10 +++++-- src/common/persistence/MemoryExtentStore.ts | 4 +-- src/queue/QueueEnvironment.ts | 4 +-- 9 files changed, 31 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index fb80ae146..a5cf08ab2 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ Following extension configurations are supported: - `azurite.skipApiVersionCheck` Skip the request API version check, by default false. - `azurite.disableProductStyleUrl` Force parsing storage account name from request Uri path, instead of from request Uri host. - `azurite.inMemoryPersistence` Disable persisting any data to disk. If the Azurite process is terminated, all data is lost. -- `azurite.extentMemoryLimit` When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of bytes. This does not limit blob, queue, or table metadata. Defaults to 50% of total memory. +- `azurite.extentMemoryLimit` When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of megabytes. This does not limit blob, queue, or table metadata. Defaults to 50% of total memory. ### [DockerHub](https://hub.docker.com/_/microsoft-azure-storage-azurite) @@ -448,16 +448,16 @@ specified. By default, the in-memory extent store (for blob and queue content) is limited to 50% of the total memory on the host machine. This is evaluated to using [`os.totalmem()`](https://nodejs.org/api/os.html#ostotalmem). This limit can be -overridden using the `--extentMemoryLimit ` option. There is no restriction on the value specified for this +overridden using the `--extentMemoryLimit ` option. There is no restriction on the value specified for this option but virtual memory may be used if the limit exceeds the amount of available physical memory as provided by the operating system. A high limit may eventually lead to out of memory errors or reduced performance. The queue and blob extent storage count towards the same limit. The `--extentMemoryLimit` setting is rejected when -`--inMemoryPersistence` is not specified. LokiJS storage (blob and queue metadata and table metadata and content) does +`--inMemoryPersistence` is not specified. LokiJS storage (blob and queue metadata and table data) does not contribute to this limit and is unbounded which is the same as without the `--inMemoryPersistence` option. ```cmd ---extentMemoryLimit +--extentMemoryLimit ``` This option is rejected when `--inMemoryPersistence` is not specified. diff --git a/docs/designs/2023-10-in-memory-persistence.md b/docs/designs/2023-10-in-memory-persistence.md index bb5c03bd1..bcb4f6381 100644 --- a/docs/designs/2023-10-in-memory-persistence.md +++ b/docs/designs/2023-10-in-memory-persistence.md @@ -80,7 +80,7 @@ A ``--inMemoryPersistence`` command line option will be introduced which will do By default, the `MemoryExtentStore` will limit itself to 50% of the total physical memory on the machine as returned by [`os.totalmem()`](https://nodejs.org/api/os.html#ostotalmem). This can be overridden using a new `--extentMemoryLimit -` option. If this option is specified without `--inMemoryPersistence`, the CLI will error out. The `os.freemem()` +` option. If this option is specified without `--inMemoryPersistence`, the CLI will error out. The `os.freemem()` will not be queried because this will lead to unpredictable behavior for the user where an extent write operation (blob content write or enqueue message) will fail intermittently based on the free memory. It is better to simple us a defined amount of memory per physical machine and potential exceed the free memory available, thus leveraging virtual memory diff --git a/package.json b/package.json index e5b7ec680..b2f418990 100644 --- a/package.json +++ b/package.json @@ -260,7 +260,7 @@ "null" ], "default": null, - "description": "When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of bytes. Defaults to 50% of total memory." + "description": "When using in-memory persistence, limit the total size of extents (blob and queue content) to a specific number of megabytes. Defaults to 50% of total memory." } } } diff --git a/src/blob/BlobEnvironment.ts b/src/blob/BlobEnvironment.ts index cd2b97f5f..bdc06f5a0 100644 --- a/src/blob/BlobEnvironment.ts +++ b/src/blob/BlobEnvironment.ts @@ -47,9 +47,9 @@ if (!(args as any).config.name) { ) .option( ["", "extentMemoryLimit"], - "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", + "Optional. The number of megabytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", -1, - s => s == -1 ? undefined : parseInt(s) + s => s == -1 ? undefined : parseFloat(s) ) .option( ["d", "debug"], diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index 37a793480..c5bf88bc0 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -13,23 +13,12 @@ export enum CertOptions { export function setExtentMemoryLimit(env: IBlobEnvironment | IQueueEnvironment) { if (env.inMemoryPersistence()) { - let limit = env.extentMemoryLimit() ?? SharedChunkStore.sizeLimit(); - if (limit && limit >= 0) { - const kb = limit / 1024; - const mb = kb / 1024; - const gb = mb / 1024; - let display; - if (gb >= 1) { - display = `${gb.toFixed(2)} GB` - } else if (mb >= 1) { - display = `${mb.toFixed(2)} MB` - } else { - display = `${kb.toFixed(2)} KB` - } - - const totalPct = Math.round(100 * limit / totalmem()) - console.log(`In-memory extent storage is enabled with a limit of ${display} (${limit} bytes, ${totalPct}% of total memory).`); - SharedChunkStore.setSizeLimit(limit); + let mb = env.extentMemoryLimit() ?? SharedChunkStore.sizeLimit(); + if (mb && mb >= 0) { + const bytes = Math.round(mb * 1024 * 1024); + const totalPct = Math.round(100 * bytes / totalmem()) + console.log(`In-memory extent storage is enabled with a limit of ${mb.toFixed(2)} MB (${bytes} bytes, ${totalPct}% of total memory).`); + SharedChunkStore.setSizeLimit(bytes); } else { console.log(`In-memory extent storage is enabled with no limit on memory used.`); } @@ -51,7 +40,7 @@ export default abstract class ConfigurationBase { public readonly pwd: string = "", public readonly oauth?: string, public readonly disableProductStyleUrl: boolean = false, - ) {} + ) { } public hasCert() { if (this.cert.length > 0 && this.key.length > 0) { @@ -92,8 +81,7 @@ export default abstract class ConfigurationBase { } public getHttpServerAddress(): string { - return `http${this.hasCert() === CertOptions.Default ? "" : "s"}://${ - this.host - }:${this.port}`; + return `http${this.hasCert() === CertOptions.Default ? "" : "s"}://${this.host + }:${this.port}`; } } diff --git a/src/common/Environment.ts b/src/common/Environment.ts index 8e0968ee3..47f0dd658 100644 --- a/src/common/Environment.ts +++ b/src/common/Environment.ts @@ -77,9 +77,9 @@ args ) .option( ["", "extentMemoryLimit"], - "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", + "Optional. The number of megabytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", -1, - s => s == -1 ? undefined : parseInt(s) + s => s == -1 ? undefined : parseFloat(s) ) .option( ["d", "debug"], diff --git a/src/common/VSCServerManagerClosedState.ts b/src/common/VSCServerManagerClosedState.ts index 82daaa5b5..ca37fce11 100644 --- a/src/common/VSCServerManagerClosedState.ts +++ b/src/common/VSCServerManagerClosedState.ts @@ -2,7 +2,7 @@ import IVSCServerManagerState from "./IVSCServerManagerState"; import VSCEnvironment from "./VSCEnvironment"; import VSCServerManagerBase from "./VSCServerManagerBase"; import { VSCServerManagerRunningState } from "./VSCServerManagerRunningState"; -import { SharedChunkStore } from "./persistence/MemoryExtentStore"; +import { DEFAULT_EXTENT_MEMORY_LIMIT, SharedChunkStore } from "./persistence/MemoryExtentStore"; export default class VSCServerManagerClosedState implements IVSCServerManagerState { @@ -14,7 +14,13 @@ export default class VSCServerManagerClosedState const limit = new VSCEnvironment().extentMemoryLimit() if (limit) { - SharedChunkStore.setSizeLimit(limit) + if (limit >= 0) { + SharedChunkStore.setSizeLimit(Math.round(limit * 1024 * 1024)) + } else { + SharedChunkStore.setSizeLimit() + } + } else { + SharedChunkStore.setSizeLimit(DEFAULT_EXTENT_MEMORY_LIMIT) } return new VSCServerManagerRunningState(); diff --git a/src/common/persistence/MemoryExtentStore.ts b/src/common/persistence/MemoryExtentStore.ts index a9ae36e6a..1f5325e35 100644 --- a/src/common/persistence/MemoryExtentStore.ts +++ b/src/common/persistence/MemoryExtentStore.ts @@ -116,8 +116,8 @@ export class MemoryExtentChunkStore { // By default, allow up to half of the total memory to be used for in-memory // extents. We don't use freemem (free memory instead of total memory) since // that would lead to a decent amount of unpredictability. -const defaultSize = Math.trunc(totalmem() * 0.5) -export const SharedChunkStore: MemoryExtentChunkStore = new MemoryExtentChunkStore(defaultSize); +export const DEFAULT_EXTENT_MEMORY_LIMIT = Math.trunc(totalmem() * 0.5) +export const SharedChunkStore: MemoryExtentChunkStore = new MemoryExtentChunkStore(DEFAULT_EXTENT_MEMORY_LIMIT); export default class MemoryExtentStore implements IExtentStore { private readonly categoryName: string; diff --git a/src/queue/QueueEnvironment.ts b/src/queue/QueueEnvironment.ts index 54669fcd6..de2c5752c 100644 --- a/src/queue/QueueEnvironment.ts +++ b/src/queue/QueueEnvironment.ts @@ -46,9 +46,9 @@ args ) .option( ["", "extentMemoryLimit"], - "Optional. The number of bytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", + "Optional. The number of megabytes to limit in-memory extent storage to. Only used with the --inMemoryPersistence option. Defaults to 50% of total memory", -1, - s => s == -1 ? undefined : parseInt(s) + s => s == -1 ? undefined : parseFloat(s) ) .option( ["d", "debug"], From d9de3730d13b6c9fbbffd00964666acc02375028 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Wed, 8 Nov 2023 11:13:06 -0500 Subject: [PATCH 35/37] Address comments --- src/azurite.ts | 3 +++ src/blob/BlobServerFactory.ts | 3 --- src/blob/main.ts | 4 +++ src/common/ConfigurationBase.ts | 31 ++++++++++++++++++----- src/common/VSCServerManagerClosedState.ts | 16 +++--------- src/queue/main.ts | 2 +- 6 files changed, 37 insertions(+), 22 deletions(-) diff --git a/src/azurite.ts b/src/azurite.ts index 1b156b7df..7861c37c3 100644 --- a/src/azurite.ts +++ b/src/azurite.ts @@ -23,6 +23,7 @@ import TableConfiguration from "./table/TableConfiguration"; import TableServer from "./table/TableServer"; import { DEFAULT_TABLE_LOKI_DB_PATH } from "./table/utils/constants"; +import { setExtentMemoryLimit } from "./common/ConfigurationBase"; // tslint:disable:no-console @@ -132,6 +133,8 @@ async function main() { // Create table server instance const tableServer = new TableServer(tableConfig); + setExtentMemoryLimit(env, true); + // Start server console.log( `Azurite Blob service is starting at ${blobConfig.getHttpServerAddress()}` diff --git a/src/blob/BlobServerFactory.ts b/src/blob/BlobServerFactory.ts index 0359e5242..a0145418d 100644 --- a/src/blob/BlobServerFactory.ts +++ b/src/blob/BlobServerFactory.ts @@ -13,7 +13,6 @@ import { DEFAULT_BLOB_LOKI_DB_PATH, DEFAULT_BLOB_PERSISTENCE_ARRAY } from "./utils/constants"; -import { setExtentMemoryLimit } from "../common/ConfigurationBase"; export class BlobServerFactory { public async createServer( @@ -91,8 +90,6 @@ export class BlobServerFactory { env.inMemoryPersistence(), ); - setExtentMemoryLimit(env); - return new BlobServer(config); } } else { diff --git a/src/blob/main.ts b/src/blob/main.ts index 1056943b2..41af28052 100644 --- a/src/blob/main.ts +++ b/src/blob/main.ts @@ -3,6 +3,8 @@ import * as Logger from "../common/Logger"; import { BlobServerFactory } from "./BlobServerFactory"; import SqlBlobServer from "./SqlBlobServer"; import BlobServer from "./BlobServer"; +import { setExtentMemoryLimit } from "../common/ConfigurationBase"; +import BlobEnvironment from "./BlobEnvironment"; // tslint:disable:no-console @@ -30,6 +32,8 @@ async function main() { // Enable debug log by default before first release for debugging purpose Logger.configLogger(config.enableDebugLog, config.debugLogFilePath); + setExtentMemoryLimit(new BlobEnvironment(), true); + // Start server console.log( `Azurite Blob service is starting on ${config.host}:${config.port}` diff --git a/src/common/ConfigurationBase.ts b/src/common/ConfigurationBase.ts index c5bf88bc0..4ea705d23 100644 --- a/src/common/ConfigurationBase.ts +++ b/src/common/ConfigurationBase.ts @@ -2,8 +2,10 @@ import * as fs from "fs"; import { OAuthLevel } from "./models"; import IBlobEnvironment from "../blob/IBlobEnvironment"; import IQueueEnvironment from "../queue/IQueueEnvironment"; -import { SharedChunkStore } from "./persistence/MemoryExtentStore"; +import { DEFAULT_EXTENT_MEMORY_LIMIT, SharedChunkStore } from "./persistence/MemoryExtentStore"; import { totalmem } from "os"; +import logger from "./Logger"; +import IEnvironment from "./IEnvironment"; export enum CertOptions { Default, @@ -11,16 +13,33 @@ export enum CertOptions { PFX } -export function setExtentMemoryLimit(env: IBlobEnvironment | IQueueEnvironment) { +export function setExtentMemoryLimit(env: IBlobEnvironment | IQueueEnvironment | IEnvironment, logToConsole: boolean) { if (env.inMemoryPersistence()) { - let mb = env.extentMemoryLimit() ?? SharedChunkStore.sizeLimit(); - if (mb && mb >= 0) { + let mb = env.extentMemoryLimit() + if (mb === undefined || typeof mb !== 'number') { + mb = DEFAULT_EXTENT_MEMORY_LIMIT / (1024 * 1024) + } + + if (mb < 0) { + throw new Error(`A negative value of '${mb}' is not allowed for the extent memory limit.`) + } + + if (mb >= 0) { const bytes = Math.round(mb * 1024 * 1024); const totalPct = Math.round(100 * bytes / totalmem()) - console.log(`In-memory extent storage is enabled with a limit of ${mb.toFixed(2)} MB (${bytes} bytes, ${totalPct}% of total memory).`); + const message = `In-memory extent storage is enabled with a limit of ${mb.toFixed(2)} MB (${bytes} bytes, ${totalPct}% of total memory).` + if (logToConsole) { + console.log(message) + } + logger.info(message) SharedChunkStore.setSizeLimit(bytes); } else { - console.log(`In-memory extent storage is enabled with no limit on memory used.`); + const message = `In-memory extent storage is enabled with no limit on memory used.` + if (logToConsole) { + console.log(message) + } + logger.info(message) + SharedChunkStore.setSizeLimit(); } } } diff --git a/src/common/VSCServerManagerClosedState.ts b/src/common/VSCServerManagerClosedState.ts index ca37fce11..200ff918d 100644 --- a/src/common/VSCServerManagerClosedState.ts +++ b/src/common/VSCServerManagerClosedState.ts @@ -1,28 +1,20 @@ +import { setExtentMemoryLimit } from "./ConfigurationBase"; import IVSCServerManagerState from "./IVSCServerManagerState"; import VSCEnvironment from "./VSCEnvironment"; import VSCServerManagerBase from "./VSCServerManagerBase"; import { VSCServerManagerRunningState } from "./VSCServerManagerRunningState"; -import { DEFAULT_EXTENT_MEMORY_LIMIT, SharedChunkStore } from "./persistence/MemoryExtentStore"; export default class VSCServerManagerClosedState implements IVSCServerManagerState { public async start( serverManager: VSCServerManagerBase ): Promise { + const env = new VSCEnvironment() + setExtentMemoryLimit(env, false) + await serverManager.createImpl(); await serverManager.startImpl(); - const limit = new VSCEnvironment().extentMemoryLimit() - if (limit) { - if (limit >= 0) { - SharedChunkStore.setSizeLimit(Math.round(limit * 1024 * 1024)) - } else { - SharedChunkStore.setSizeLimit() - } - } else { - SharedChunkStore.setSizeLimit(DEFAULT_EXTENT_MEMORY_LIMIT) - } - return new VSCServerManagerRunningState(); } diff --git a/src/queue/main.ts b/src/queue/main.ts index ff0984390..1e96b4b9d 100644 --- a/src/queue/main.ts +++ b/src/queue/main.ts @@ -78,7 +78,7 @@ async function main() { // Create server instance const server = new QueueServer(config); - setExtentMemoryLimit(env); + setExtentMemoryLimit(env, true); // Start server console.log( From 803e27cdd1310de6641c264e71448ef53caf9195 Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 14 Nov 2023 10:15:48 -0500 Subject: [PATCH 36/37] Update README --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index a5cf08ab2..b3ee7ab82 100644 --- a/README.md +++ b/README.md @@ -452,6 +452,15 @@ overridden using the `--extentMemoryLimit ` option. There is no restr option but virtual memory may be used if the limit exceeds the amount of available physical memory as provided by the operating system. A high limit may eventually lead to out of memory errors or reduced performance. +As blob or queue content (i.e. bytes in the in-memory extent store) is deleted, the memory is not freed immediately. +Similar to the default file-system based extent store, both the blob and queue service have an extent garbage collection +(GC) process. This process is in addition to the standard Node.js runtime GC. The extent GC periodically detects unused +extents and deletes them from the extent store. This happens on a regular time period rather than immediately after +the blob or queue REST API operation the caused some content to be deleted. This means that process memory consumed by +the deleted blob or queue content will only be released after both the extent GC and the runtime GC have run. The extent +GC will remove the reference to the in-memory byte storage and the runtime GC will free the unreferenced memory some +time after that. The blob extent GC runs every 10 minutes and the queue extent GC runs every 1 minute. + The queue and blob extent storage count towards the same limit. The `--extentMemoryLimit` setting is rejected when `--inMemoryPersistence` is not specified. LokiJS storage (blob and queue metadata and table data) does not contribute to this limit and is unbounded which is the same as without the `--inMemoryPersistence` option. From 9b913056a5c9d93e05fde8b5236a211748ca1b0c Mon Sep 17 00:00:00 2001 From: Joel Verhagen Date: Tue, 14 Nov 2023 10:18:08 -0500 Subject: [PATCH 37/37] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b3ee7ab82..d6896860a 100644 --- a/README.md +++ b/README.md @@ -456,7 +456,7 @@ As blob or queue content (i.e. bytes in the in-memory extent store) is deleted, Similar to the default file-system based extent store, both the blob and queue service have an extent garbage collection (GC) process. This process is in addition to the standard Node.js runtime GC. The extent GC periodically detects unused extents and deletes them from the extent store. This happens on a regular time period rather than immediately after -the blob or queue REST API operation the caused some content to be deleted. This means that process memory consumed by +the blob or queue REST API operation that caused some content to be deleted. This means that process memory consumed by the deleted blob or queue content will only be released after both the extent GC and the runtime GC have run. The extent GC will remove the reference to the in-memory byte storage and the runtime GC will free the unreferenced memory some time after that. The blob extent GC runs every 10 minutes and the queue extent GC runs every 1 minute.