Merge branch 'release-v0.11.0'

.travis.yml

@@ -1,10 +1,25 @@
 language: node_js
-node_js:
-  - "6"
-  - "8"
-  - "10"
 install: npm install
-script:  let "n = 0";npm run lint; let "n = n + $?";npm run ci-test; let "n = n + $?";(exit $n)
+
+jobs:
+    include:
+    - stage: tests
+      name: "Unit + Integration Node 6"
+      script: npm run test
+      node_js: "6"
+    - script: npm run test
+      name: "Unit + Integration Node 8"
+      node_js: "8"
+    - script: npm run test
+      name: "Unit + Integration Node 10"
+      node_js: "10"
+
+    - script: npm run lint
+      name: "Linting"
+      node_js: "10"
+    - script: npm run ci-test
+      name: "Coverage"
+      node_js: "10"
 
 notifications:
     webhooks:

CHANGELOG.md

@@ -1,6 +1,63 @@
-Changes in 0.10.1 (2018-07-30)
+Changes in 0.11.0 (2018-08-28)
+==============================
+
+No changes since previous RC, see below for full list of changes
+
+Changes in 0.11.0-rc4 (2018-08-24)
+==============================
+
+Bug Fixes:
+
+* Fixed a bug where content of events the bridge hadn't cached
+  were not being used in replies.
+
+Changes in 0.11.0-rc3 (2018-08-24)
 ==============================
 
+- The bridge now depends on matrix-appservice-bridge 1.6.0c
+
+Bug Fixes:
+
+* We were calling authedRequest but the request was not mocked out.
+
+Changes in 0.11.0-rc2 (2018-08-24)
+==============================
+
+- The bridge now depends on matrix-appservice-bridge 1.6.0b
+
+Bug Fixes:
+
+* There was a bug involving intents in m-a-b so it was bumped
+
+Changes in 0.11.0-rc1 (2018-08-23)
+==============================
+
+- The bridge now depends on matrix-appservice-bridge 1.6.0a
+
+New features & improvements:
+* Cache modes internally #630
+* Replace nicks with user pill mentions #650 #658
+* Kick users if we fail to create an IRC client for them on join (aka ILINE kicks) #639
+* SASL support #643
+* Add err_nononreg so we can announce PMs that failed #645
+* Formatting of replies #647
+
+Bug Fixes:
+* Fix invalidchar nick #655
+* Don't answer any msgtypes other than text in an admin room. #642
+* Fix provisoner leaving users on unlink #649
+
+Metrics:
+* Metrics for MatrixHandler - Iline Kicks #644
+* Idle connection metrics #651
+* QueuePool.waitingItems should use it's internal queue size #656
+
+Misc:
+* Section out tests, linting and coverage into seperate stages for Travis #657
+
+Changes in 0.10.1 (2018-07-30)
+==============================
+
  - Missed a few changes from master
 
 Changes in 0.10.0 (2018-07-30)

config.sample.yaml

@@ -63,6 +63,9 @@ ircService:
       ssl: true
       # Whether or not IRC server is using a self-signed cert or not providing CA Chain
       sslselfsign: false
+      # Should the connection attempt to identify via SASL (if a server or user password is given)
+      # If false, this will use PASS instead. If SASL fails, we do not fallback to PASS.
+      sasl: false
       # Whether to allow expired certs when connecting to the IRC server.
       # Usually this should be off. Default: false.
       allowExpiredCerts: false
@@ -73,7 +76,7 @@ ircService:
       #  -----END CERTIFICATE-----
 
       #
-      # The connection password to send for all clients as a PASS command. Optional.
+      # The connection password to send for all clients as a PASS (or SASL, if enabled above) command. Optional.
       # password: 'pa$$w0rd'
       #
       # Whether or not to send connection/error notices to real Matrix users. Default: true.
@@ -374,8 +377,15 @@ ircService:
   # Optional. Enable Prometheus metrics. If this is enabled, you MUST install `prom-client`:
   #   $ npm install [email protected]
   # Metrics will then be available via GET /metrics on the bridge listening port (-p).
-  # metrics:
-  #   enabled: true
+  metrics:
+    # Whether to actually enable the metric endpoint. Default: false
+    enabled: true
+    # When collecting remote user active times, which "buckets" should be used. Defaults are given below.
+    # The bucket name is formed of a duration and a period. (h=hours,d=days,w=weeks).
+    remoteUserAgeBuckets:
+      - "1h"
+      - "1d"
+      - "1w"
 
   # The nedb database URI to connect to. This is the name of the directory to
   # dump .db files to. This is relative to the project directory.
@@ -425,3 +435,8 @@ ircService:
   # `!storepass server.name passw0rd. When a connection is made to IRC on behalf of
   # the Matrix user, this password will be sent as the server password (PASS command).
   passwordEncryptionKeyPath: "passkey.pem"
+
+  # Config for Matrix -> IRC bridging
+  matrixHandler:
+    # Cache this many matrix events in memory to be used for m.relates_to messages (usually replies).
+    eventCacheSize: 4096

lib/DataStore.js

@@ -311,6 +311,46 @@ DataStore.prototype.getMappingsForChannelByOrigin = function(server, channel, or
     });
 };
 
+DataStore.prototype.getModesForChannel = function (server, channel) {
+    log.info("getModesForChannel (server=%s, channel=%s)",
+        server.domain, channel
+    );
+    let remoteId = IrcRoom.createId(server, channel);
+    return this._roomStore.getEntriesByRemoteId(remoteId).then((entries) => {
+        const mapping = {};
+        entries.forEach((entry) => {
+            mapping[entry.matrix.getId()] = entry.remote.get("modes") || [];
+        });
+        return mapping;
+    });
+};
+
+DataStore.prototype.setModeForRoom = Promise.coroutine(function*(roomId, mode, enabled=True) {
+    log.info("setModeForRoom (mode=%s, roomId=%s, enabled=%s)",
+        mode, roomId, enabled
+    );
+    return this._roomStore.getEntriesByMatrixId(roomId).then((entries) => {
+        entries.map((entry) => {
+            const modes = entry.remote.get("modes") || [];
+            const hasMode = modes.includes(mode);
+
+            if (hasMode === enabled) {
+                return;
+            }
+            if (enabled) {
+                modes.push(mode);
+            }
+            else {
+                modes.splice(modes.indexOf(mode), 1);
+            }
+
+            entry.remote.set("modes", modes);
+
+            this._roomStore.upsertEntry(entry);
+        });
+    });
+});
+
 DataStore.prototype.setPmRoom = function(ircRoom, matrixRoom, userId, virtualUserId) {
     log.info("setPmRoom (id=%s, addr=%s chan=%s real=%s virt=%s)",
         matrixRoom.getId(), ircRoom.server.domain, ircRoom.channel, userId,

lib/bridge/IrcBridge.js

@@ -48,7 +48,7 @@ function IrcBridge(config, registration) {
     this.joinedRoomList = [];
 
     // Dependency graph
-    this.matrixHandler = new MatrixHandler(this);
+    this.matrixHandler = new MatrixHandler(this, this.config.matrixHandler);
     this.ircHandler = new IrcHandler(this);
     this._clientPool = new ClientPool(this);
     var dirPath = this.config.ircService.databaseUri.substring("nedb://".length);
@@ -125,6 +125,14 @@ IrcBridge.prototype._initialiseMetrics = function() {
     const metrics = this._bridge.getPrometheusMetrics();
 
     this._bridge.registerBridgeGauges(() => {
+        const remoteUsersByAge = new AgeCounters(
+            this.config.ircService.metrics.remoteUserAgeBuckets || ["1h", "1d", "1w"]
+        );
+
+        this.ircServers.forEach((server) => {
+            this._clientPool.updateActiveConnectionMetrics(server.domain, remoteUsersByAge);
+        });
+
         return {
             // TODO(paul): actually fill these in
             matrixRoomConfigs: 0,
@@ -140,7 +148,7 @@ IrcBridge.prototype._initialiseMetrics = function() {
             remoteRoomsByAge: zeroAge,
 
             matrixUsersByAge: zeroAge,
-            remoteUsersByAge: zeroAge,
+            remoteUsersByAge,
         };
     });
 
@@ -181,11 +189,23 @@ IrcBridge.prototype._initialiseMetrics = function() {
         help: "Track calls made to the IRC Handler",
         labels: ["method"]
     });
+
+    const matrixHandlerConnFailureKicks = metrics.addCounter({
+        name: "matrixhandler_connection_failure_kicks",
+        help: "Track IRC connection failures resulting in kicks",
+        labels: ["server"]
+    });
+
     metrics.addCollector(() => {
         this.ircServers.forEach((server) => {
             reconnQueue.set({server: server.domain},
                 this._clientPool.totalReconnectsWaiting(server.domain)
             );
+            let mxMetrics = this.matrixHandler.getMetrics(server.domain);
+            matrixHandlerConnFailureKicks.inc(
+                {server: server.domain},
+                mxMetrics["connection_failure_kicks"] || 0
+            );
         });
 
         Object.keys(this.memberListSyncers).forEach((server) => {

lib/bridge/IrcHandler.js

@@ -14,6 +14,14 @@ var QuitDebouncer = require("./QuitDebouncer.js");
 
 const JOIN_DELAY_MS = 250;
 const JOIN_DELAY_CAP_MS = 30 * 60 * 1000; // 30 mins
+const MODES_TO_WATCH = [
+    "m", // We want to ensure we do not miss rooms that get unmoderated.
+    "k",
+    "i",
+    "s"
+];
+
+const NICK_USERID_CACHE_MAX = 512;
 
 function IrcHandler(ircBridge) {
     this.ircBridge = ircBridge;
@@ -44,6 +52,8 @@ function IrcHandler(ircBridge) {
         //'$fromUserId $toUserId' : Promise
     };
 
+    this.nickUserIdMapCache = new Map(); // server:channel => mapping
+
     // Map<string,bool> which contains nicks we know have been registered/has display name
     this._registeredNicks = Object.create(null);
     this.getMetrics();
@@ -462,6 +472,21 @@ IrcHandler.prototype.onMessage = Promise.coroutine(function*(req, server, fromUs
     );
 
     let mxAction = MatrixAction.fromIrcAction(action);
+    let mapping;
+    if (this.nickUserIdMapCache.has(`${server.domain}:${channel}`)) {
+        mapping = this.nickUserIdMapCache.get(`${server.domain}:${channel}`);
+    }
+    else {
+        mapping = this.ircBridge.getClientPool().getNickUserIdMappingForChannel(
+            server, channel
+        );
+        this.nickUserIdMapCache.set(`${server.domain}:${channel}`, mapping);
+        if (this.nickUserIdMapCache.size > NICK_USERID_CACHE_MAX) {
+            this.nickUserIdMapCache.delete(this.nickUserIdMapCache.keys()[0]);
+        }
+    }
+
+    yield mxAction.formatMentions(mapping, this.ircBridge.getAppServiceBridge().getIntent());
 
     if (!mxAction) {
         req.log.error("Couldn't map IRC action to matrix action");
@@ -513,6 +538,8 @@ IrcHandler.prototype.onJoin = Promise.coroutine(function*(req, server, joiningUs
         this.incrementMetric("join");
     }
 
+    this._invalidateNickUserIdMap(server, chan);
+
     let nick = joiningUser.nick;
     let syncType = kind === "names" ? "initial" : "incremental";
     if (!server.shouldSyncMembershipToMatrix(syncType, chan)) {
@@ -672,6 +699,7 @@ IrcHandler.prototype.onKick = Promise.coroutine(function*(req, server,
  */
 IrcHandler.prototype.onPart = Promise.coroutine(function*(req, server, leavingUser, chan, kind) {
     this.incrementMetric("part");
+    this._invalidateNickUserIdMap(server, chan);
     // parts are always incremental (only NAMES are initial)
     if (!server.shouldSyncMembershipToMatrix("incremental", chan)) {
         req.log.info("Server doesn't mirror parts.");
@@ -824,6 +852,7 @@ IrcHandler.prototype._onModeratedChannelToggle = Promise.coroutine(function*(req
                 "onModeratedChannelToggle: (channel=%s,enabled=%s) power levels updated in room %s",
                 channel, enabled, roomId
             );
+            this.ircBridge.getStore().setModeForRoom(roomId, "m", enabled);
         }
         catch (err) {
             req.log.error("Failed to alter power level in room %s : %s", roomId, err);
@@ -859,11 +888,18 @@ IrcHandler.prototype._onPrivateMode = Promise.coroutine(function*(req, server, c
         }
         const key = this.ircBridge.publicitySyncer.getIRCVisMapKey(server.getNetworkId(), channel);
 
+        matrixRooms.map((room) => {
+            this.ircBridge.getStore().setModeForRoom(room.getId(), "s", enabled);
+        });
         // Update the visibility for all rooms connected to this channel
         return this.ircBridge.publicitySyncer.updateVisibilityMap(
             true, key, enabled
         );
     }
+    // "k" and "i"
+    matrixRooms.map((room) => {
+        this.ircBridge.getStore().setModeForRoom(room.getId(), mode, enabled);
+    });
 
     var promises = matrixRooms.map((room) => {
         switch (mode) {
@@ -882,7 +918,7 @@ IrcHandler.prototype._onPrivateMode = Promise.coroutine(function*(req, server, c
                 return this._setMatrixRoomAsInviteOnly(room, enabled);
             default:
                 // Not reachable, but warn anyway in case of future additions
-                log.warn(`onMode: Unhandled channel mode ${mode}`);
+                req.log.warn(`onMode: Unhandled channel mode ${mode}`);
                 return Promise.resolve();
         }
     });
@@ -911,10 +947,25 @@ IrcHandler.prototype.onModeIs = Promise.coroutine(function*(req, server, channel
         }
     );
 
-    // If the channel does not have 's' as part of its mode, trigger the equivalent of '-s'
-    if (mode.indexOf('s') === -1) {
-        promises.push(this.onMode(req, server, channel, 'onModeIs function', 's', false));
-    }
+    // We cache modes per room, so extract the set of modes for all these rooms.
+    const roomModeMap = yield this.ircBridge.getStore().getModesForChannel(server, channel);
+    const oldModes = new Set();
+    Object.values(roomModeMap).forEach((roomMode) => {
+        roomMode.forEach((m) => {oldModes.add(m)});
+    });
+    req.log.debug(`Got cached mode for ${channel} ${[...oldModes]}`);
+
+    // For each cached mode we have for the room, that is no longer set: emit a disabled mode.
+    promises.concat([...oldModes].map((oldModeChar) => {
+        if (!MODES_TO_WATCH.includes(oldModeChar)) {
+            return Promise.resolve();
+        }
+        req.log.debug(`${server.domain} ${channel}: Checking if '${oldModeChar}' is still set.`);
+        if (!mode.includes(oldModeChar)) { // If the mode is no longer here.
+            req.log.debug(`${oldModeChar} has been unset, disabling.`);
+            return this.onMode(req, server, channel, 'onModeIs function', oldModeChar, false);
+        }
+    }));
 
     yield Promise.all(promises);
 });
@@ -974,6 +1025,10 @@ IrcHandler.prototype._setMatrixRoomAsInviteOnly = function(room, isInviteOnly) {
     );
 };
 
+IrcHandler.prototype._invalidateNickUserIdMap = function(server, channel) {
+    this.nickUserIdMapCache.delete(`${server.domain}:${channel}`);
+}
+
 IrcHandler.prototype.incrementMetric = function(metric) {
     if (this._callCountMetrics[metric] === undefined) {
         this._callCountMetrics[metric] = 0;