Merge pull request #18 from pryv/1.9.0-alpha

1.9.0 beta

.api-version

@@ -1 +1 @@
-1.9.0-10-open
+1.9.0-open

.gitignore

@@ -2,7 +2,7 @@ var-pryv/
 public_html/
 app-web-auth3/
 node_modules/
-rec.la-certificates/
+docker/dockerized-open-pryv
 
 # not commiting this into OS version
 

.licenser.yml

@@ -0,0 +1,82 @@
+files:
+  "**/*.js":
+    header:
+      startBlock: |
+        /**
+         * @license
+      linePrefix: " * "
+      endBlock: " */"
+
+  "**/package.json":
+    json:
+      force:
+        author: "{AUTHOR_NAME} <{AUTHOR_EMAIL}> ({AUTHOR_WEB})"
+        license: "{SPDX}"
+        private: true
+      defaults:
+        homepage: "{HOMEPAGE}"
+        description: "{DESCRIPTION}"
+      sortPackage: true
+
+    siblingLicenseFile:
+      name: "LICENSE"
+
+  "**/README.md":
+    footer:
+      startBlock: "\n\n# License\n\n"
+      linePrefix: ""
+      endBlock: ""
+      license: "[{SPDX}](LICENSE)"
+
+ignore:
+- .git
+- .vscode
+- coverage
+- node_modules
+- test-results
+- var-pryv
+- service-mail/node_modules
+- app-web-auth3
+
+substitutions:
+  YEARS:
+    start: 2020
+    end: CURRENT_YEAR
+  AUTHOR_NAME: "Pryv S.A."
+  AUTHOR_EMAIL: [email protected]
+  AUTHOR_WEB: https://pryv.com
+  DESCRIPTION: "This package is part of Open Pryv.io, a Pryv S.A. software"
+  HOMEPAGE: https://pryv.com
+  SPDX: BSD-3-Clause
+
+license: |
+  Copyright (C) {YEARS} {AUTHOR_NAME} {AUTHOR_WEB} 
+
+  This file is part of Open-Pryv.io and released under BSD-Clause-3 License
+
+  Redistribution and use in source and binary forms, with or without 
+  modification, are permitted provided that the following conditions are met:
+
+  1. Redistributions of source code must retain the above copyright notice, 
+    this list of conditions and the following disclaimer.
+
+  2. Redistributions in binary form must reproduce the above copyright notice, 
+    this list of conditions and the following disclaimer in the documentation 
+    and/or other materials provided with the distribution.
+
+  3. Neither the name of the copyright holder nor the names of its contributors 
+    may be used to endorse or promote products derived from this software 
+    without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 
+  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 
+  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+  SPDX-License-Identifier: BSD-3-Clause

README.md

@@ -50,15 +50,14 @@ Choose your Set-up
   - Native installation
 - Launch Pryv.io on a server exposed to the Internet with built-in SSL, this requires to have a hostname pointing to the public IP of your server.
   - Download docker images (quick start)
-  - Launch image with Open Pryv.io installed on Exoscale hosting provider: [link to guide](https://api.pryv.com/image-exoscale-open-pryv.io/) (quick start including hosting)
   - Native installation
 - Launch Pryv.io on a server with an external SSL termination. You know what you are doing.
   - Download docker images
   - Native installation
 
 ### Docker
 
-The dockerized versions and their instructions are available at this link: [Download link](https://api.pryv.com/open-pryv.io/docker/dockerized-open-pryv.io.tgz).
+The dockerized versions and their instructions are available at this link: [Download link](https://api.pryv.com/open-pryv.io/docker/dockerized-open-pryv-1.9.0.io.tgz).
 
 If you wish to build the images yourself, refer to the following README: [docker/README-build.md](docker/README-build.md).
 
@@ -69,59 +68,50 @@ Once it is running, you can continue with the [tutorials](#start).
 *Prerequisites*:
 
 - Node v18.14.2 [Node.js home page](https://nodejs.org/)
-- [just](https://github.com/casey/just#installation)
 
 The installation script has been tested on Linux Ubuntu 18.04 LTS and MacOSX.
 
-1. `just setup-dev-env` to setup local file structure and install MongoDB
-2. `just install [--no-optional]` to install node modules
+1. `npm run setup-dev-env` to setup local file structure and install MongoDB
+2. `npm install` to install node modules
 
-#### Native setup with external SSL
+#### Native setup with no SSL
 
 [setup the environment](#native)
 
-- `npm run pryv` - mail and database logs will be kept in `var-pryv/logs/local-*.log`
-
 Each service independently - logs will be displayed on the console
 
 - `npm run database` start mongodb
 - `npm run api` start the API server on port 3000 (default)
 - `npm run mail` start the mail service
 
-#### Local native setup
-
-[setup the environment](#native)
-
-- `npm run local` is the equivalent of running `npm run pryv` + `npm run proxy` using `configs/rec-la.yml`. This setup is useful to test Open Pryv.io locally.
+#### Local native setup with rec.la loopback SSL
 
-- `npm run proxy` (with database) and `npm run proxied` (without database) based on [rec-la](https://github.com/pryv/rec-la), it will expose the server running on http://localhost:3000 with an SSL certificate on https://my-computer.rec.la:4443 in this case you need to edit `configs/rec-la.yml`.
-
-Note: if rec.la certificate are expired you can refresh them with ./scripts/update-recla-certificates
-
-#### Native Server setup with built-in SSL
+[rec.la](https://rec.la) certificates facilitate local developpment by enabling https on localhost. 
 
 [setup the environment](#native)
+- `npm run database` to start mongodb 
+- (optional) `npm run mail` start the mail service
+- `npm run apirecla` to start api server using `configs/api-recla.yml`
 
-1. Run `npm run pryv` to start the API
-2. Configure NGINX and certificate
+You can now access you API from you own computer with SSL on 
+- `https://my-computer.rec.la:4443`
 
-You can find a NGINX configuration that you can include in your `sites-enabled/` in [configs/site.conf](configs/site.conf).
+You can check by opening [https://my-computer.rec.la:4443/reg/service/info](https://my-computer.rec.la:4443/reg/service/info)
 
-You must change `${HOSTNAME}` to match the hostname of the public URL.
+And create new users or access token from the [Pryv Access Token Generation Page](https://api.pryv.com/app-web-access/?pryvServiceInfoUrl=https://l.rec.la:4443/reg/service/info)
 
-##### SSL certificate
 
-Using [certbot](https://certbot.eff.org/), you can generate a SSL certificate for your platform using `sudo certbot --nginx -d ${HOSTNAME}`.
+#### Native setup with custom SSL
 
-To set an automatic renewal, run `crontab -e` and append the following line:
+[setup the environment](#native)
 
-```cron
-0 12 * * * /usr/bin/certbot renew --quiet
-```
+1. Edit `http:ssl` part in `./configs/api.yml` file to point to your certificates an key files.
+2. Update `dnsLess:publicUrl` in `./configs/api.yml` to match 
+3. Run `npm run pryv` to start the API
 
 ### Config
 
-For the native installation, edit `config.yml`, otherwise `docker/local/dockerized-config.yml`:
+For the native installation, edit `./configs/api.yml`
 
 ```yaml
 dnsLess:
@@ -130,7 +120,7 @@ http:
   port: 3000
   ip: 127.0.0.1
 auth:
-  adminAccessKey: iuahwd0ba87hw0bd7a8hwd
+  adminAccessKey: REPLACE_ME 
   trustedApps: "*@https://pryv.github.io*, *@https://*.rec.la*"
 eventFiles:
   attachmentsDirPath: var-pryv/attachment-files
@@ -157,8 +147,6 @@ services:
   - **trustedApps** list of web apps that can be trusted-app functionalities
      API for trusted apps: [API reference](https://api.pryv.com/reference/)
     see: [SETUP Guide - customize authentication](https://api.pryv.com/customer-resources/pryv.io-setup/#customize-authentication-registration-and-reset-password-apps)
-- **eventFiles**
-  - **attachmentsDirPath** Directory where event attachment files will be stored on the file system.
 - **service** [API documentation on Service Information](https://api.pryv.com/reference/#service-info)
 - **services:email** see [Options & Customization](#custom-email) below
 
@@ -242,7 +230,7 @@ Contributions are welcome. Get in touch with the Pryv team or submit a PR with y
 
 ## License
 
-Copyright (c) 2020 Pryv S.A. https://pryv.com
+Copyright (c) 2019-2023 Pryv S.A. https://pryv.com
 
 This file is part of Open-Pryv.io and released under BSD-Clause-3 License
 

components/api-server/src/server.js

@@ -35,6 +35,9 @@
 // Always require application first to be sure boiler is initialized
 const { getApplication } = require('api-server/src/application');
 const http = require('http');
+const https = require('https');
+const fs = require('fs');
+const recLaOptionsAsync = require('rec.la').httpsOptionsAsync;
 const { axonMessaging } = require('messages');
 const { pubsub } = require('messages');
 const { getUsersRepository } = require('business/src/users');
@@ -75,9 +78,36 @@ class Server {
     // register API methods
     await this.registerApiMethods();
     // Setup HTTP and register server; setup Socket.IO.
-    const server = http.createServer(app.expressApp);
+    let server = null;
+    const serverInfos = {
+      hostname: null
+    };
+    if (config.get('http:ssl:rec.la')) { // SSL is used in openSource version
+      await new Promise((resolve, reject) => {
+        recLaOptionsAsync((err, recLaOptions) => {
+          if (err) return reject(err);
+          server = https.createServer(recLaOptions, app.expressApp);
+          serverInfos.hostname = 'my-computer.rec.la';
+          resolve();
+        });
+      });
+      this.logger.info('SSL Mode using rec.la certificates');
+    } else if (config.get('http:ssl:keyFile')) { // https with local files
+      const options = {
+        key: fs.readFileSync(config.get('http:ssl:keyFile')),
+        cert: fs.readFileSync(config.get('http:ssl:certFile'))
+      };
+      if (config.get('http:ssl:caFile')) {
+        options.ca = [fs.readFileSync(config.get('http:ssl:caFile'))];
+      }
+      server = https.createServer(options, app.expressApp);
+      serverInfos.hostname = 'custom-according-to-your-ssl-cert';
+      this.logger.info('SSL Mode using custom certificates');
+    } else { // http
+      server = http.createServer(app.expressApp);
+    }
     await this.setupSocketIO(server);
-    await this.startListen(server);
+    await this.startListen(server, serverInfos);
     if (!this.isOpenSource) {
       await this.setupReporting();
     }
@@ -139,7 +169,7 @@ class Server {
    * @param {http.Server} server
    * @returns {Promise<void>}
    */
-  async startListen (server) {
+  async startListen (server, info = {}) {
     const config = this.config;
     const logger = this.logger;
     const port = config.get('http:port');
@@ -171,8 +201,9 @@ class Server {
       });
     });
     const address = server.address();
-    const protocol = 'http';
-    const serverUrl = protocol + '://' + address.address + ':' + address.port;
+    const protocol = server.key == null ? 'http' : 'https';
+    const hostnameStr = info.hostname || address.address;
+    const serverUrl = protocol + '://' + hostnameStr + ':' + address.port;
     logger.debug('listening on ' + serverUrl);
     logger.info(`Core Server (API module) listening on ${serverUrl}`);
     // Warning if ignoring forbidden updates

components/storage/src/migrations/1.9.0.js

@@ -48,6 +48,7 @@ module.exports = async function (context, callback) {
   await SystemStreamsSerializer.init();
   try {
     await moveAttachments();
+    await migratePasswords(context);
     await migrateHistory(context);
   } catch (e) {
     return callback(e);
@@ -78,6 +79,22 @@ async function moveAttachments () {
   }
 }
 
+async function migratePasswords (context) {
+  const logger = getLogger('migration-1.9.0:passwords');
+  const userAccountStorage = await require('storage').getUserAccountStorage();
+  const query = { streamIds: { $in: [':_system:passwordHash'] } };
+  const eventsCollection = await context.database.getCollection({
+    name: 'events'
+  });
+  const cursor = await eventsCollection.find(query, { projection: { _id: 1, userId: 1, content: 1, created: 1, createdBy: 1 } });
+  while (await cursor.hasNext()) {
+    const event = await cursor.next();
+    await userAccountStorage.addPasswordHash(event.userId, event.content, event.createdBy || 'system', event.created);
+    await eventsCollection.deleteMany({ userId: event.userId, _id: event._id });
+    logger.info('Migrating password for userId: ' + event.userId);
+  }
+}
+
 async function migrateHistory (context) {
   const logger = getLogger('migration-1.9.0:historical-events');
   const eventsCollection = await context.database.getCollection({
@@ -110,7 +127,10 @@ async function migrateHistory (context) {
       }
     };
     requests.push(request);
-    if (requests.length > BUFFER_SIZE) { requests = await flushToDb(requests, eventsCollection); }
+    if (requests.length > BUFFER_SIZE) {
+      requests = [];
+      await flushToDb(requests, eventsCollection);
+    }
   }
   await flushToDb(requests, eventsCollection);