Feature/16 limit free collections

docs-src/docs/releases/16.0.0.md

@@ -71,9 +71,10 @@ I completely rewrote them and improved performance especially on initial load wh
 - If the handler of a [RxPipeline](../rx-pipeline.md) throws an error, block the whole pipeline and emit the error to the outside.
 - Throw error when dexie.js RxStorage is used with optional index fields [#6643](https://github.com/pubkey/rxdb/pull/6643#issuecomment-2505310082).
 
-## Minor things
+## Other
 
 - Added more [performance tests](https://rxdb.info/rx-storage-performance.html)
+- The amount of collections in the open source version has been limited to `16`.
 - Moved RxQuery checks into dev-mode.
 - RxQuery.remove() now internally does a bulk operation for better performance.
 - Lazily process bulkWrite() results for less CPU usage.

docs-src/docs/rx-collection.md

@@ -316,3 +316,14 @@ const is = isRxCollection(myObj);
     You have to call the `addCollections()` method each time you create your database. This will create the JavaScript object instance of the RxCollection so that you can use it in the RxDatabase. The persisted data will be automatically in your RxCollection each time you create it.
     </div>
 </details>
+<details>
+    <summary>How to remove the limit of 16 collections?</summary>
+    <div>
+    In the open-source version of RxDB, the amount of RxCollections that can exist in parallel is limited to `16`.
+    To remove this limit, you can purchase the [Premium Plugins](/premium) and call the `setPremiumFlag()` function before creating a database:
+    ```ts
+    import { setPremiumFlag } from 'rxdb-premium/plugins/shared';
+    setPremiumFlag();
+    ```
+    </div>
+</details>

docs-src/src/pages/premium-submitted.tsx

@@ -18,8 +18,13 @@ export default function PremiumSubmitted() {
             const database = await getDatabase();
             const formValueDoc = await database.getLocal<FormValueDocData>(FORM_VALUE_DOCUMENT_ID);
             console.dir(formValueDoc);
+
+            window.trigger(
+                'premium_lead',
+                Math.floor(formValueDoc._data.data.price / 3) // assume lead-to-sale-rate is 33%.
+            );
             if (!formValueDoc) {
-                window.trigger('premium_lead_unknown', 300);
+                window.trigger('premium_lead_unknown', 0);
                 await database.upsertLocal<FormValueDocData>(FORM_VALUE_DOCUMENT_ID, {
                     formSubmitted: true
                 });
@@ -30,9 +35,9 @@ export default function PremiumSubmitted() {
                 }
                 window.trigger(
                     'premium_lead_' + formValueDoc._data.data.homeCountry.toLowerCase(),
-                    Math.floor(formValueDoc._data.data.price / 3) // assume lead-to-sale-rate is 33%.
+                    0, // zero because we track the value already at the previous event trigger
                 );
-                await formValueDoc.incrementalPatch({formSubmitted: true});
+                await formValueDoc.incrementalPatch({ formSubmitted: true });
             }
         })();
     });

src/plugins/dev-mode/dev-mode-tracking.ts

@@ -1,9 +1,7 @@
 import type { RxDatabase } from '../../types';
 import {
-    PREMIUM_FLAG_HASH,
-    RXDB_UTILS_GLOBAL,
     RXDB_VERSION,
-    defaultHashSha256
+    hasPremiumFlag
 } from '../utils/index.ts';
 
 
@@ -29,11 +27,7 @@ export async function addDevModeTrackingIframe(db: RxDatabase) {
 
 
     // do not show if premium flag is set.
-    if (
-        RXDB_UTILS_GLOBAL.premium &&
-        typeof RXDB_UTILS_GLOBAL.premium === 'string' &&
-        (await defaultHashSha256(RXDB_UTILS_GLOBAL.premium) === PREMIUM_FLAG_HASH)
-    ) {
+    if (await hasPremiumFlag()) {
         return;
     }
 

src/plugins/dev-mode/error-messages.ts

@@ -4,6 +4,8 @@
  * This is mainly because error-string are hard to compress and we need a smaller build
  */
 
+import { NON_PREMIUM_COLLECTION_LIMIT } from '../utils/utils-premium.ts';
+
 
 export const ERROR_MESSAGES = {
     // util.js / config
@@ -102,6 +104,7 @@ export const ERROR_MESSAGES = {
     COL21: 'The RxCollection is closed or removed already, either from this JavaScript realm or from another, like a browser tab',
     CONFLICT: 'Document update conflict. When changing a document you must work on the previous revision',
     COL22: '.bulkInsert() and .bulkUpsert() cannot be run with multiple documents that have the same primary key',
+    COL23: 'In the open-source version of RxDB, the amount of collections that can exist in parallel is limited to '+NON_PREMIUM_COLLECTION_LIMIT+'. If you already purchased the premium access, you can remove this limit: https://rxdb.info/rx-collection.html#faq',
 
     // rx-document.js
     DOC1: 'RxDocument.get$ cannot get observable of in-array fields because order cannot be guessed',

src/plugins/storage-dexie/rx-storage-instance-dexie.ts

@@ -7,7 +7,8 @@ import {
     ensureNotFalsy,
     defaultHashSha256,
     RXDB_UTILS_GLOBAL,
-    PREMIUM_FLAG_HASH
+    PREMIUM_FLAG_HASH,
+    hasPremiumFlag
 } from '../utils/index.ts';
 import type {
     RxStorageInstance,
@@ -82,11 +83,7 @@ export class RxStorageInstanceDexie<RxDocType> implements RxStorageInstance<
 
         if (
             !shownNonPremiumLog &&
-            (
-                !RXDB_UTILS_GLOBAL.premium ||
-                typeof RXDB_UTILS_GLOBAL.premium !== 'string' ||
-                (await defaultHashSha256(RXDB_UTILS_GLOBAL.premium) !== PREMIUM_FLAG_HASH)
-            )
+            !(await hasPremiumFlag())
         ) {
             console.warn(
                 [

src/plugins/utils/index.ts

@@ -16,3 +16,4 @@ export * from './utils-time.ts';
 export * from './utils-other.ts';
 export * from './utils-rxdb-version.ts';
 export * from './utils-global.ts';
+export * from './utils-premium.ts';

src/plugins/utils/utils-global.ts

@@ -3,6 +3,3 @@
  * can be imported and mutated at will.
  */
 export const RXDB_UTILS_GLOBAL: any = {};
-
-
-export const PREMIUM_FLAG_HASH = '6da4936d1425ff3a5c44c02342c6daf791d266be3ae8479b8ec59e261df41b93';

src/plugins/utils/utils-premium.ts

@@ -0,0 +1,37 @@
+import { RXDB_UTILS_GLOBAL } from './utils-global.ts';
+import { defaultHashSha256 } from './utils-hash.ts';
+import { PROMISE_RESOLVE_FALSE } from './utils-promise.ts';
+
+export const PREMIUM_FLAG_HASH = '6da4936d1425ff3a5c44c02342c6daf791d266be3ae8479b8ec59e261df41b93';
+export const NON_PREMIUM_COLLECTION_LIMIT = 16;
+
+let hasPremiumPromise: Promise<boolean> = PROMISE_RESOLVE_FALSE;
+let premiumChecked = false;
+
+/**
+ * Here we check if the premium flag has been set.
+ * This code exists in the open source version of RxDB.
+ * Yes you are allowed to fork the repo and just overwrite this function.
+ * However you might better spend this time developing your real project
+ * and supporting the RxDB efforts by buying premium.
+ */
+export async function hasPremiumFlag() {
+    if (premiumChecked) {
+        return hasPremiumPromise;
+    }
+    premiumChecked = true;
+
+    hasPremiumPromise = (async () => {
+        if (
+            RXDB_UTILS_GLOBAL.premium &&
+            typeof RXDB_UTILS_GLOBAL.premium === 'string' &&
+            (await defaultHashSha256(RXDB_UTILS_GLOBAL.premium) === PREMIUM_FLAG_HASH)
+        ) {
+            return true;
+        } else {
+            return false;
+        }
+    })();
+
+    return hasPremiumPromise;
+}

src/rx-collection.ts

@@ -12,7 +12,9 @@ import {
     ensureNotFalsy,
     getFromMapOrThrow,
     PROMISE_RESOLVE_FALSE,
-    PROMISE_RESOLVE_VOID
+    PROMISE_RESOLVE_VOID,
+    NON_PREMIUM_COLLECTION_LIMIT,
+    hasPremiumFlag
 } from './plugins/utils/index.ts';
 import {
     fillObjectDataBeforeInsert,
@@ -112,6 +114,8 @@ const HOOKS_KEYS = ['insert', 'save', 'remove', 'create'] as const;
 type HookKeyType = typeof HOOKS_KEYS[number];
 let hooksApplied = false;
 
+export const OPEN_COLLECTIONS = new Set<RxCollectionBase<any, any, any>>();
+
 export class RxCollectionBase<
     InstanceCreationOptions,
     RxDocumentType = { [prop: string]: any; },
@@ -157,6 +161,17 @@ export class RxCollectionBase<
                 filter(changeEventBulk => changeEventBulk.collectionName === this.name)
             );
         } else { }
+
+
+        /**
+         * Must be last because the hooks might throw on dev-mode
+         * checks and we do not want to have broken collections here.
+         * RxCollection instances created for testings do not have a database
+         * so we do not add these to the list.
+         */
+        if (this.database) {
+            OPEN_COLLECTIONS.add(this);
+        }
     }
 
     get insert$(): Observable<RxChangeEventInsert<RxDocumentType>> {
@@ -215,6 +230,35 @@ export class RxCollectionBase<
     public onRemove: (() => MaybePromise<any>)[] = [];
 
     public async prepare(): Promise<void> {
+
+        if (!(await hasPremiumFlag())) {
+
+            /**
+             * When used in a test suite, we often open and close many databases with collections
+             * while not awaiting the database.close() call to improve the test times.
+             * So when reopening collections and the OPEN_COLLECTIONS size is full,
+             * we retry after some times to account for this.
+             */
+            let count = 0;
+            while (count < 10 && OPEN_COLLECTIONS.size > NON_PREMIUM_COLLECTION_LIMIT) {
+                count++;
+                await this.promiseWait(30);
+            }
+            if (OPEN_COLLECTIONS.size > NON_PREMIUM_COLLECTION_LIMIT) {
+                throw newRxError('COL23', {
+                    database: this.database.name,
+                    collection: this.name,
+                    args: {
+                        existing: Array.from(OPEN_COLLECTIONS.values()).map(c => ({
+                            db: c.database ? c.database.name : '',
+                            c: c.name
+                        }))
+                    }
+                });
+            }
+        }
+
+
         this.storageInstance = getWrappedStorageInstance(
             this.database,
             this.internalStorageInstance,
@@ -880,6 +924,8 @@ export class RxCollectionBase<
             return PROMISE_RESOLVE_FALSE;
         }
 
+        OPEN_COLLECTIONS.delete(this);
+
 
         await Promise.all(this.onClose.map(fn => fn()));
 
@@ -1114,6 +1160,7 @@ export function createRxCollection(
              * we yet have to close the storage instances.
              */
             .catch(err => {
+                OPEN_COLLECTIONS.delete(collection);
                 return storageInstance.close()
                     .then(() => Promise.reject(err as Error));
             });

test/unit/attachments.test.ts

@@ -28,8 +28,14 @@ import {
     createBlobFromBase64,
     createBlob,
     blobToString,
-    RxDocumentWriteData
+    RxDocumentWriteData,
+    addRxPlugin
 } from '../../plugins/core/index.mjs';
+import { RxDBMigrationSchemaPlugin } from '../../plugins/migration-schema/index.mjs';
+addRxPlugin(RxDBMigrationSchemaPlugin);
+import { RxDBUpdatePlugin } from '../../plugins/update/index.mjs';
+addRxPlugin(RxDBUpdatePlugin);
+
 
 const STATIC_FILE_SERVER_URL = 'http://localhost:18001/';
 

test/unit/hooks.test.ts

@@ -19,8 +19,8 @@ import {
 } from '../../plugins/core/index.mjs';
 
 
-describeParallel('hooks.test.js', () => {
-    describe('get/set', () => {
+describe('hooks.test.js', () => {
+    describeParallel('get/set', () => {
         it('should set a hook', async () => {
             const c = await humansCollection.create(0);
             c.preSave(function () { }, false);
@@ -43,7 +43,7 @@ describeParallel('hooks.test.js', () => {
             c.database.close();
         });
     });
-    describe('insert', () => {
+    describeParallel('insert', () => {
         describe('pre', () => {
             describe('positive', () => {
                 it('series', async () => {
@@ -182,7 +182,7 @@ describeParallel('hooks.test.js', () => {
             });
         });
     });
-    describe('save', () => {
+    describeParallel('save', () => {
         describe('pre', () => {
             describe('positive', () => {
                 it('series', async () => {
@@ -308,7 +308,7 @@ describeParallel('hooks.test.js', () => {
             describe('negative', () => { });
         });
     });
-    describe('remove', () => {
+    describeParallel('remove', () => {
         describe('pre', () => {
             describe('positive', () => {
                 it('series', async () => {
@@ -469,7 +469,7 @@ describeParallel('hooks.test.js', () => {
             describe('negative', () => { });
         });
     });
-    describe('postCreate', () => {
+    describeParallel('postCreate', () => {
         describe('positive', () => {
             it('should define a getter', async () => {
                 const db = await createRxDatabase({
@@ -523,7 +523,7 @@ describeParallel('hooks.test.js', () => {
             });
         });
     });
-    describe('issues', () => {
+    describeParallel('issues', () => {
         it('ISSUE #158 : Throwing error in async preInsert does not prevent insert', async () => {
             const c = await humansCollection.create(0);
             c.preInsert(async function () {

test/unit/last.test.ts

@@ -3,7 +3,8 @@ import { waitUntil } from 'async-test-util';
 import {
     dbCount,
     BROADCAST_CHANNEL_BY_TOKEN,
-    getFromMapOrThrow
+    getFromMapOrThrow,
+    OPEN_COLLECTIONS
 } from '../../plugins/core/index.mjs';
 import config from './config.ts';
 
@@ -42,6 +43,18 @@ describe('last.test.ts (' + config.storage.name + ')', () => {
     it('ensure every db is cleaned up', () => {
         assert.strictEqual(dbCount(), 0);
     });
+    it('ensure all collections are closed', async () => {
+        try {
+            await waitUntil(() => {
+                return OPEN_COLLECTIONS.size === 0;
+            }, 5 * 1000);
+        } catch (err) {
+            const openCollections = Array.from(OPEN_COLLECTIONS.values()).map(c => ({ c: c.name, db: c.database ? c.database.name : '' }));
+            console.log('open collectios:');
+            console.dir(openCollections);
+            throw new Error('not all collections have been closed (' + openCollections.length + ')');
+        }
+    });
     it('ensure all BroadcastChannels are closed', async () => {
         try {
             await waitUntil(() => {