diff --git a/.travis.yml b/.travis.yml index 0bb609dda..b75b99bd3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,25 @@ language: node_js -node_js: - - "6" - - "8" - - "10" install: npm install -script: let "n = 0";npm run lint; let "n = n + $?";npm run ci-test; let "n = n + $?";(exit $n) + +jobs: + include: + - stage: tests + name: "Unit + Integration Node 6" + script: npm run test + node_js: "6" + - script: npm run test + name: "Unit + Integration Node 8" + node_js: "8" + - script: npm run test + name: "Unit + Integration Node 10" + node_js: "10" + + - script: npm run lint + name: "Linting" + node_js: "10" + - script: npm run ci-test + name: "Coverage" + node_js: "10" notifications: webhooks: diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cf28d414..3f3f79386 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,63 @@ -Changes in 0.10.1 (2018-07-30) +Changes in 0.11.0 (2018-08-28) +============================== + +No changes since previous RC, see below for full list of changes + +Changes in 0.11.0-rc4 (2018-08-24) +============================== + +Bug Fixes: + +* Fixed a bug where content of events the bridge hadn't cached + were not being used in replies. + +Changes in 0.11.0-rc3 (2018-08-24) ============================== +- The bridge now depends on matrix-appservice-bridge 1.6.0c + +Bug Fixes: + +* We were calling authedRequest but the request was not mocked out. + +Changes in 0.11.0-rc2 (2018-08-24) +============================== + +- The bridge now depends on matrix-appservice-bridge 1.6.0b + +Bug Fixes: + +* There was a bug involving intents in m-a-b so it was bumped + +Changes in 0.11.0-rc1 (2018-08-23) +============================== + +- The bridge now depends on matrix-appservice-bridge 1.6.0a + +New features & improvements: +* Cache modes internally #630 +* Replace nicks with user pill mentions #650 #658 +* Kick users if we fail to create an IRC client for them on join (aka ILINE kicks) #639 +* SASL support #643 +* Add err_nononreg so we can announce PMs that failed #645 +* Formatting of replies #647 + +Bug Fixes: +* Fix invalidchar nick #655 +* Don't answer any msgtypes other than text in an admin room. #642 +* Fix provisoner leaving users on unlink #649 + +Metrics: +* Metrics for MatrixHandler - Iline Kicks #644 +* Idle connection metrics #651 +* QueuePool.waitingItems should use it's internal queue size #656 + +Misc: +* Section out tests, linting and coverage into seperate stages for Travis #657 + +Changes in 0.10.1 (2018-07-30) +============================== + - Missed a few changes from master Changes in 0.10.0 (2018-07-30) diff --git a/config.sample.yaml b/config.sample.yaml index 9b6db9bd6..2256e21d1 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -63,6 +63,9 @@ ircService: ssl: true # Whether or not IRC server is using a self-signed cert or not providing CA Chain sslselfsign: false + # Should the connection attempt to identify via SASL (if a server or user password is given) + # If false, this will use PASS instead. If SASL fails, we do not fallback to PASS. + sasl: false # Whether to allow expired certs when connecting to the IRC server. # Usually this should be off. Default: false. allowExpiredCerts: false @@ -73,7 +76,7 @@ ircService: # -----END CERTIFICATE----- # - # The connection password to send for all clients as a PASS command. Optional. + # The connection password to send for all clients as a PASS (or SASL, if enabled above) command. Optional. # password: 'pa$$w0rd' # # Whether or not to send connection/error notices to real Matrix users. Default: true. @@ -374,8 +377,15 @@ ircService: # Optional. Enable Prometheus metrics. If this is enabled, you MUST install `prom-client`: # $ npm install prom-client@6.3.0 # Metrics will then be available via GET /metrics on the bridge listening port (-p). - # metrics: - # enabled: true + metrics: + # Whether to actually enable the metric endpoint. Default: false + enabled: true + # When collecting remote user active times, which "buckets" should be used. Defaults are given below. + # The bucket name is formed of a duration and a period. (h=hours,d=days,w=weeks). + remoteUserAgeBuckets: + - "1h" + - "1d" + - "1w" # The nedb database URI to connect to. This is the name of the directory to # dump .db files to. This is relative to the project directory. @@ -425,3 +435,8 @@ ircService: # `!storepass server.name passw0rd. When a connection is made to IRC on behalf of # the Matrix user, this password will be sent as the server password (PASS command). passwordEncryptionKeyPath: "passkey.pem" + + # Config for Matrix -> IRC bridging + matrixHandler: + # Cache this many matrix events in memory to be used for m.relates_to messages (usually replies). + eventCacheSize: 4096 \ No newline at end of file diff --git a/lib/DataStore.js b/lib/DataStore.js index a46feec44..864bb763f 100644 --- a/lib/DataStore.js +++ b/lib/DataStore.js @@ -311,6 +311,46 @@ DataStore.prototype.getMappingsForChannelByOrigin = function(server, channel, or }); }; +DataStore.prototype.getModesForChannel = function (server, channel) { + log.info("getModesForChannel (server=%s, channel=%s)", + server.domain, channel + ); + let remoteId = IrcRoom.createId(server, channel); + return this._roomStore.getEntriesByRemoteId(remoteId).then((entries) => { + const mapping = {}; + entries.forEach((entry) => { + mapping[entry.matrix.getId()] = entry.remote.get("modes") || []; + }); + return mapping; + }); +}; + +DataStore.prototype.setModeForRoom = Promise.coroutine(function*(roomId, mode, enabled=True) { + log.info("setModeForRoom (mode=%s, roomId=%s, enabled=%s)", + mode, roomId, enabled + ); + return this._roomStore.getEntriesByMatrixId(roomId).then((entries) => { + entries.map((entry) => { + const modes = entry.remote.get("modes") || []; + const hasMode = modes.includes(mode); + + if (hasMode === enabled) { + return; + } + if (enabled) { + modes.push(mode); + } + else { + modes.splice(modes.indexOf(mode), 1); + } + + entry.remote.set("modes", modes); + + this._roomStore.upsertEntry(entry); + }); + }); +}); + DataStore.prototype.setPmRoom = function(ircRoom, matrixRoom, userId, virtualUserId) { log.info("setPmRoom (id=%s, addr=%s chan=%s real=%s virt=%s)", matrixRoom.getId(), ircRoom.server.domain, ircRoom.channel, userId, diff --git a/lib/bridge/IrcBridge.js b/lib/bridge/IrcBridge.js index fba8b2b5c..fa09f0351 100644 --- a/lib/bridge/IrcBridge.js +++ b/lib/bridge/IrcBridge.js @@ -48,7 +48,7 @@ function IrcBridge(config, registration) { this.joinedRoomList = []; // Dependency graph - this.matrixHandler = new MatrixHandler(this); + this.matrixHandler = new MatrixHandler(this, this.config.matrixHandler); this.ircHandler = new IrcHandler(this); this._clientPool = new ClientPool(this); var dirPath = this.config.ircService.databaseUri.substring("nedb://".length); @@ -125,6 +125,14 @@ IrcBridge.prototype._initialiseMetrics = function() { const metrics = this._bridge.getPrometheusMetrics(); this._bridge.registerBridgeGauges(() => { + const remoteUsersByAge = new AgeCounters( + this.config.ircService.metrics.remoteUserAgeBuckets || ["1h", "1d", "1w"] + ); + + this.ircServers.forEach((server) => { + this._clientPool.updateActiveConnectionMetrics(server.domain, remoteUsersByAge); + }); + return { // TODO(paul): actually fill these in matrixRoomConfigs: 0, @@ -140,7 +148,7 @@ IrcBridge.prototype._initialiseMetrics = function() { remoteRoomsByAge: zeroAge, matrixUsersByAge: zeroAge, - remoteUsersByAge: zeroAge, + remoteUsersByAge, }; }); @@ -181,11 +189,23 @@ IrcBridge.prototype._initialiseMetrics = function() { help: "Track calls made to the IRC Handler", labels: ["method"] }); + + const matrixHandlerConnFailureKicks = metrics.addCounter({ + name: "matrixhandler_connection_failure_kicks", + help: "Track IRC connection failures resulting in kicks", + labels: ["server"] + }); + metrics.addCollector(() => { this.ircServers.forEach((server) => { reconnQueue.set({server: server.domain}, this._clientPool.totalReconnectsWaiting(server.domain) ); + let mxMetrics = this.matrixHandler.getMetrics(server.domain); + matrixHandlerConnFailureKicks.inc( + {server: server.domain}, + mxMetrics["connection_failure_kicks"] || 0 + ); }); Object.keys(this.memberListSyncers).forEach((server) => { diff --git a/lib/bridge/IrcHandler.js b/lib/bridge/IrcHandler.js index 5ab6c8d01..f7496d900 100644 --- a/lib/bridge/IrcHandler.js +++ b/lib/bridge/IrcHandler.js @@ -14,6 +14,14 @@ var QuitDebouncer = require("./QuitDebouncer.js"); const JOIN_DELAY_MS = 250; const JOIN_DELAY_CAP_MS = 30 * 60 * 1000; // 30 mins +const MODES_TO_WATCH = [ + "m", // We want to ensure we do not miss rooms that get unmoderated. + "k", + "i", + "s" +]; + +const NICK_USERID_CACHE_MAX = 512; function IrcHandler(ircBridge) { this.ircBridge = ircBridge; @@ -44,6 +52,8 @@ function IrcHandler(ircBridge) { //'$fromUserId $toUserId' : Promise }; + this.nickUserIdMapCache = new Map(); // server:channel => mapping + // Map which contains nicks we know have been registered/has display name this._registeredNicks = Object.create(null); this.getMetrics(); @@ -462,6 +472,21 @@ IrcHandler.prototype.onMessage = Promise.coroutine(function*(req, server, fromUs ); let mxAction = MatrixAction.fromIrcAction(action); + let mapping; + if (this.nickUserIdMapCache.has(`${server.domain}:${channel}`)) { + mapping = this.nickUserIdMapCache.get(`${server.domain}:${channel}`); + } + else { + mapping = this.ircBridge.getClientPool().getNickUserIdMappingForChannel( + server, channel + ); + this.nickUserIdMapCache.set(`${server.domain}:${channel}`, mapping); + if (this.nickUserIdMapCache.size > NICK_USERID_CACHE_MAX) { + this.nickUserIdMapCache.delete(this.nickUserIdMapCache.keys()[0]); + } + } + + yield mxAction.formatMentions(mapping, this.ircBridge.getAppServiceBridge().getIntent()); if (!mxAction) { req.log.error("Couldn't map IRC action to matrix action"); @@ -513,6 +538,8 @@ IrcHandler.prototype.onJoin = Promise.coroutine(function*(req, server, joiningUs this.incrementMetric("join"); } + this._invalidateNickUserIdMap(server, chan); + let nick = joiningUser.nick; let syncType = kind === "names" ? "initial" : "incremental"; if (!server.shouldSyncMembershipToMatrix(syncType, chan)) { @@ -672,6 +699,7 @@ IrcHandler.prototype.onKick = Promise.coroutine(function*(req, server, */ IrcHandler.prototype.onPart = Promise.coroutine(function*(req, server, leavingUser, chan, kind) { this.incrementMetric("part"); + this._invalidateNickUserIdMap(server, chan); // parts are always incremental (only NAMES are initial) if (!server.shouldSyncMembershipToMatrix("incremental", chan)) { req.log.info("Server doesn't mirror parts."); @@ -824,6 +852,7 @@ IrcHandler.prototype._onModeratedChannelToggle = Promise.coroutine(function*(req "onModeratedChannelToggle: (channel=%s,enabled=%s) power levels updated in room %s", channel, enabled, roomId ); + this.ircBridge.getStore().setModeForRoom(roomId, "m", enabled); } catch (err) { req.log.error("Failed to alter power level in room %s : %s", roomId, err); @@ -859,11 +888,18 @@ IrcHandler.prototype._onPrivateMode = Promise.coroutine(function*(req, server, c } const key = this.ircBridge.publicitySyncer.getIRCVisMapKey(server.getNetworkId(), channel); + matrixRooms.map((room) => { + this.ircBridge.getStore().setModeForRoom(room.getId(), "s", enabled); + }); // Update the visibility for all rooms connected to this channel return this.ircBridge.publicitySyncer.updateVisibilityMap( true, key, enabled ); } + // "k" and "i" + matrixRooms.map((room) => { + this.ircBridge.getStore().setModeForRoom(room.getId(), mode, enabled); + }); var promises = matrixRooms.map((room) => { switch (mode) { @@ -882,7 +918,7 @@ IrcHandler.prototype._onPrivateMode = Promise.coroutine(function*(req, server, c return this._setMatrixRoomAsInviteOnly(room, enabled); default: // Not reachable, but warn anyway in case of future additions - log.warn(`onMode: Unhandled channel mode ${mode}`); + req.log.warn(`onMode: Unhandled channel mode ${mode}`); return Promise.resolve(); } }); @@ -911,10 +947,25 @@ IrcHandler.prototype.onModeIs = Promise.coroutine(function*(req, server, channel } ); - // If the channel does not have 's' as part of its mode, trigger the equivalent of '-s' - if (mode.indexOf('s') === -1) { - promises.push(this.onMode(req, server, channel, 'onModeIs function', 's', false)); - } + // We cache modes per room, so extract the set of modes for all these rooms. + const roomModeMap = yield this.ircBridge.getStore().getModesForChannel(server, channel); + const oldModes = new Set(); + Object.values(roomModeMap).forEach((roomMode) => { + roomMode.forEach((m) => {oldModes.add(m)}); + }); + req.log.debug(`Got cached mode for ${channel} ${[...oldModes]}`); + + // For each cached mode we have for the room, that is no longer set: emit a disabled mode. + promises.concat([...oldModes].map((oldModeChar) => { + if (!MODES_TO_WATCH.includes(oldModeChar)) { + return Promise.resolve(); + } + req.log.debug(`${server.domain} ${channel}: Checking if '${oldModeChar}' is still set.`); + if (!mode.includes(oldModeChar)) { // If the mode is no longer here. + req.log.debug(`${oldModeChar} has been unset, disabling.`); + return this.onMode(req, server, channel, 'onModeIs function', oldModeChar, false); + } + })); yield Promise.all(promises); }); @@ -974,6 +1025,10 @@ IrcHandler.prototype._setMatrixRoomAsInviteOnly = function(room, isInviteOnly) { ); }; +IrcHandler.prototype._invalidateNickUserIdMap = function(server, channel) { + this.nickUserIdMapCache.delete(`${server.domain}:${channel}`); +} + IrcHandler.prototype.incrementMetric = function(metric) { if (this._callCountMetrics[metric] === undefined) { this._callCountMetrics[metric] = 0; diff --git a/lib/bridge/MatrixHandler.js b/lib/bridge/MatrixHandler.js index ab215b657..743892422 100644 --- a/lib/bridge/MatrixHandler.js +++ b/lib/bridge/MatrixHandler.js @@ -17,7 +17,14 @@ const MSG_PMS_DISABLED = "[Bridge] Sorry, PMs are disabled on this bridge."; const MSG_PMS_DISABLED_FEDERATION = "[Bridge] Sorry, PMs are disabled on " + "this bridge over federation."; -function MatrixHandler(ircBridge) { +const KICK_RETRY_DELAY_MS = 15000; +const KICK_DELAY_JITTER = 30000; +/* Number of events to store in memory for use in replies. */ +const DEFAULT_EVENT_CACHE_SIZE = 4096; +/* Length of the source text in a formatted reply message */ +const REPLY_SOURCE_MAX_LENGTH = 32; + +function MatrixHandler(ircBridge, config) { this.ircBridge = ircBridge; // maintain a list of room IDs which are being processed invite-wise. This is // required because invites are processed asyncly, so you could get invite->msg @@ -27,6 +34,13 @@ function MatrixHandler(ircBridge) { }; this._memberTracker = null; + this._eventCache = new Map(); //eventId => {body, sender} + config = config || {} + this._eventCacheMaxSize = config.eventCacheSize === undefined ? + DEFAULT_EVENT_CACHE_SIZE : config.eventCacheSize; + this.metrics = { + //domain => {"metricname" => value} + }; } // ===== Matrix Invite Handling ===== @@ -215,34 +229,36 @@ MatrixHandler.prototype._onAdminMessage = Promise.coroutine(function*(req, event if (cmd === "!help") { let helpCommands = { "!join": { - example: `!join irc.example.com #channel [key]`, + example: `!join [irc.example.net] #channel [key]`, summary: `Join a channel (with optional channel key)`, }, "!nick": { - example: `!nick irc.example.com DesiredNick`, + example: `!nick [irc.example.net] DesiredNick`, summary: "Change your nick. If no arguments are supplied, " + "your current nick is shown.", }, "!whois": { - example: `!whois NickName|@alice:matrix.org`, + example: `!whois [irc.example.net] NickName|@alice:matrix.org`, summary: "Do a /whois lookup. If a Matrix User ID is supplied, " + "return information about that user's IRC connection.", }, "!storepass": { - example: `!storepass [irc.example.com] passw0rd`, + example: `!storepass [irc.example.net] passw0rd`, summary: `Store a NickServ password (server password)`, }, "!removepass": { - example: `!removepass [irc.example.com]`, + example: `!removepass [irc.example.net]`, summary: `Remove a previously stored NickServ password`, }, "!quit": { example: `!quit`, - summary: `Leave all bridged channels and remove your connection to IRC`, + summary: "Leave all bridged channels, on all networks, and remove your " + + "connections to all networks.", }, "!cmd": { - example: `!cmd [irc.server] COMMAND [arg0 [arg1 [...]]]`, - summary: `Issue a raw IRC command. These will not produce a reply.`, + example: `!cmd [irc.example.net] COMMAND [arg0 [arg1 [...]]]`, + summary: "Issue a raw IRC command. These will not produce a reply." + + "(Note that the command must be all uppercase.)", }, }; @@ -853,9 +869,36 @@ MatrixHandler.prototype._onJoin = Promise.coroutine(function*(req, event, user) } // get the virtual IRC user for this user promises.push(Promise.coroutine(function*() { - let bridgedClient = yield self.ircBridge.getBridgedClient( - room.server, user.getId(), (event.content || {}).displayname - ); + let bridgedClient; + let kickIntent; + try { + bridgedClient = yield self.ircBridge.getBridgedClient( + room.server, user.getId(), (event.content || {}).displayname + ); + } + catch (e) { + // We need to kick on failure to get a client. + req.log.info(`${user.getId()} failed to get a IRC connection. Kicking from room.`); + kickIntent = self.ircBridge.getAppServiceBridge().getIntent(); + } + + while (kickIntent) { + try { + yield kickIntent.kick( + event.room_id, user.getId(), + `Connection limit reached for ${room.server.domain}. Please try again later.` + ); + this._incrementMetric(room.server.domain, "connection_failure_kicks"); + break; + } + catch (err) { + const delay = KICK_RETRY_DELAY_MS + (Math.random() * KICK_DELAY_JITTER); + req.log.warn( + `User was not kicked. Retrying in ${delay}ms. ${err}` + ); + yield Promise.delay(delay); + } + } // Check for a displayname change and update nick accordingly. if (event.content.displayname !== bridgedClient.displayName) { @@ -869,9 +912,15 @@ MatrixHandler.prototype._onJoin = Promise.coroutine(function*(req, event, user) if (room.server.allowsNickChanges() && config.getDesiredNick() === null ) { - bridgedClient.changeNick( - room.server.getNick(bridgedClient.userId, event.content.displayname), - false); + try { + const newNick = room.server.getNick( + bridgedClient.userId, event.content.displayname + ); + bridgedClient.changeNick(newNick, false); + } + catch (e) { + req.log.warn(`Didn't change nick on the IRC side: ${e}`); + } } } @@ -1115,7 +1164,8 @@ MatrixHandler.prototype._onMessage = Promise.coroutine(function*(req, event) { let ircAction = IrcAction.fromMatrixAction(mxAction); let ircRooms = yield this.ircBridge.getStore().getIrcChannelsForRoomId(event.room_id); - if (ircRooms.length === 0) { + // Sometimes bridge's message each other and get stuck in a silly loop. Ensure it's m.text + if (ircRooms.length === 0 && event.content && event.content.msgtype === "m.text") { // could be an Admin room, so check. let adminRoom = yield this.ircBridge.getStore().getAdminRoomById(event.room_id); if (!adminRoom) { @@ -1220,16 +1270,37 @@ MatrixHandler.prototype._onMessage = Promise.coroutine(function*(req, event) { MatrixHandler.prototype._sendIrcAction = Promise.coroutine( function*(req, ircRoom, ircClient, ircAction, event) { - // Send the action as is if it is not a text message - // Also, check for the existance of the getSplitMessages method. - if (event.content.msgtype !== "m.text" || - !(ircClient.unsafeClient && ircClient.unsafeClient.getSplitMessages)) { + if (event.content.msgtype !== "m.text") { yield this.ircBridge.sendIrcAction(ircRoom, ircClient, ircAction); return; } let text = event.content.body; + let cacheBody = text; + if (event.content["m.relates_to"] && event.content["m.relates_to"]["m.in_reply_to"]) { + const reply = yield this._textForReplyEvent(event, ircRoom); + if (reply !== undefined) { + ircAction.text = text = reply.formatted; + cacheBody = reply.reply; + } + } + this._eventCache.set(event.event_id, { + body: cacheBody.substr(0, REPLY_SOURCE_MAX_LENGTH), + sender: event.sender + }); + + // Cache events in here so we can refer to them for replies. + if (this._eventCache.size > this._eventCacheMaxSize) { + const delKey = this._eventCache.entries().next().value[0]; + this._eventCache.delete(delKey); + } + + // Check for the existance of the getSplitMessages method. + if (!(ircClient.unsafeClient && ircClient.unsafeClient.getSplitMessages)) { + yield this.ircBridge.sendIrcAction(ircRoom, ircClient, ircAction); + return; + } // Generate an array of individual messages that would be sent let potentialMessages = ircClient.unsafeClient.getSplitMessages(ircRoom.channel, text); @@ -1450,6 +1521,103 @@ MatrixHandler.prototype._onUserQuery = Promise.coroutine(function*(req, userId) yield this.ircBridge.getMatrixUser(ircUser); }); +MatrixHandler.prototype._textForReplyEvent = Promise.coroutine(function*(event, ircRoom) { + const REPLY_REGEX = /> <(@.*:.*)>(.*)\n\n(.*)/; + const REPLY_NAME_MAX_LENGTH = 12; + const eventId = event.content["m.relates_to"]["m.in_reply_to"].event_id; + const match = REPLY_REGEX.exec(event.content.body); + if (match.length !== 4) { + return; + } + + let rplName; + let rplSource; + const rplText = match[3]; + if (!this._eventCache.has(eventId)) { + // Fallback to fetching from the homeserver. + try { + const eventContent = yield this.ircBridge.getAppServiceBridge().getIntent().getEvent( + event.room_id, eventId + ); + rplName = eventContent.sender; + if (typeof(eventContent.content.body) !== "string") { + throw Error("'body' was not a string."); + } + const isReply = eventContent.content["m.relates_to"] && + eventContent.content["m.relates_to"]["m.in_reply_to"]; + if (isReply) { + const sourceMatch = REPLY_REGEX.exec(eventContent.content.body); + rplSource = sourceMatch.length === 4 ? sourceMatch[3] : event.content.body; + } + else { + rplSource = eventContent.content.body; + } + rplSource = rplSource.substr(0, REPLY_SOURCE_MAX_LENGTH); + this._eventCache.set(eventId, {sender: rplName, body: rplSource}); + } + catch (err) { + // If we couldn't find the event, then frankly we can't + // trust it and we won't treat it as a reply. + return { + formatted: rplText, + reply: rplText, + }; + } + } + else { + rplName = this._eventCache.get(eventId).sender; + rplSource = this._eventCache.get(eventId).body; + } + + // Get the first non-blank line from the source. + const lines = rplSource.split('\n').filter((line) => !/^\s*$/.test(line)) + if (lines.length > 0) { + // Cut to a maximum length. + rplSource = lines[0].substr(0, REPLY_SOURCE_MAX_LENGTH); + // Ellipsis if needed. + if (lines[0].length > REPLY_SOURCE_MAX_LENGTH) { + rplSource = rplSource + "..."; + } + // Wrap in formatting + rplSource = ` "${rplSource}"`; + } + else { + // Don't show a source because we couldn't format one. + rplSource = ""; + } + + // Fetch the sender's IRC nick. + const sourceClient = this.ircBridge.getIrcUserFromCache(ircRoom.server, rplName); + if (sourceClient) { + rplName = sourceClient.nick; + } + else { + // Somehow we failed, so fallback to localpart. + rplName = rplName.substr(1, + Math.min(REPLY_NAME_MAX_LENGTH, rplName.indexOf(":") - 1) + ); + } + + return { + formatted: `<${rplName}${rplSource}> ${rplText}`, + reply: rplText, + }; +}); + +MatrixHandler.prototype._incrementMetric = function(serverDomain, metricName) { + let metricSet = this.metrics[serverDomain]; + if (!metricSet) { + metricSet = this.metrics[serverDomain] = {}; + } + if (metricSet[metricName] === undefined) { + metricSet[metricName] = 1; + } + else { + metricSet[metricName]++; + } + this.metrics[serverDomain] = metricSet; +} + // EXPORTS MatrixHandler.prototype.onMemberEvent = function(req, event, inviter, invitee) { @@ -1484,6 +1652,12 @@ MatrixHandler.prototype.onUserQuery = function(req, userId) { return reqHandler(req, this._onUserQuery(req, userId)) }; +MatrixHandler.prototype.getMetrics = function(serverDomain) { + const metrics = this.metrics[serverDomain] || {}; + this.metrics[serverDomain] = {} + return metrics || {}; +} + function reqHandler(req, promise) { return promise.then(function(res) { req.resolve(res); diff --git a/lib/bridge/MemberListSyncer.js b/lib/bridge/MemberListSyncer.js index 83e81edc6..af2116958 100644 --- a/lib/bridge/MemberListSyncer.js +++ b/lib/bridge/MemberListSyncer.js @@ -427,6 +427,14 @@ MemberListSyncer.prototype.getUsersWaitingToLeave = function() { return this._usersToLeave; } +MemberListSyncer.prototype.addToLeavePool = function(userIds, roomId, channel) { + this._usersToLeave += userIds.length; + this._leaveQueuePool.enqueue(roomId + " " + channel, { + roomId, + userIds + }); +} + function getRoomMemberData(server, roomId, stateEvents, appServiceUserId) { stateEvents = stateEvents || []; var data = { diff --git a/lib/config/schema.yml b/lib/config/schema.yml index 7abfa19ab..4ca77f907 100644 --- a/lib/config/schema.yml +++ b/lib/config/schema.yml @@ -25,6 +25,11 @@ properties: properties: enabled: type: "boolean" + remoteUserAgeBuckets: + type: "array" + items: + type: "string" + pattern: "^[0-9]+(h|d|w)$" statsd: type: "object" properties: @@ -78,6 +83,11 @@ properties: type: "number" passwordEncryptionKeyPath: type: "string" + matrixHandler: + type: "object" + properties: + eventCacheSize: + type: "integer" servers: type: "object" # all properties must follow the following @@ -94,6 +104,8 @@ properties: type: "boolean" sslselfsign: type: "boolean" + sasl: + type: "boolean" allowExpiredCerts: type: "boolean" password: diff --git a/lib/irc/BridgedClient.js b/lib/irc/BridgedClient.js index 014f2c6e7..016e10388 100644 --- a/lib/irc/BridgedClient.js +++ b/lib/irc/BridgedClient.js @@ -175,12 +175,14 @@ BridgedClient.prototype.connect = Promise.coroutine(function*() { } }); connInst.client.addListener("error", (err) => { + // Errors we MUST notify the user about, regardless of the bridge's admin room config. + const ERRORS_TO_FORCE = ["err_nononreg"] if (!err || !err.command || connInst.dead) { return; } var msg = "Received an error on " + this.server.domain + ": " + err.command + "\n"; msg += JSON.stringify(err.args); - this._eventBroker.sendMetadata(this, msg); + this._eventBroker.sendMetadata(this, msg, ERRORS_TO_FORCE.includes(err.command)); }); return connInst; } @@ -489,7 +491,7 @@ BridgedClient.prototype._getValidNick = function(nick, throwOnInvalid) { // strip illegal chars according to RFC 2812 Sect 2.3.1 - let n = nick.replace(/[^A-Za-z0-9\]\[\^\\\{\}\-`_\|]/g, ""); + let n = nick.replace(BridgedClient.illegalCharactersRegex, ""); if (throwOnInvalid && n !== nick) { throw new Error(`Nick '${nick}' contains illegal characters.`); } @@ -736,4 +738,6 @@ BridgedClient.prototype._joinChannel = function(channel, key, attemptCount) { return defer.promise; } +BridgedClient.illegalCharactersRegex = /[^A-Za-z0-9\]\[\^\\\{\}\-`_\|]/g; + module.exports = BridgedClient; diff --git a/lib/irc/ClientPool.js b/lib/irc/ClientPool.js index 71e557a4d..ad30f1fd6 100644 --- a/lib/irc/ClientPool.js +++ b/lib/irc/ClientPool.js @@ -246,7 +246,28 @@ ClientPool.prototype.totalReconnectsWaiting = function (serverDomain) { return this._reconnectQueues[serverDomain].waitingItems; } return 0; -} +}; + +ClientPool.prototype.updateActiveConnectionMetrics = function(server, ageCounter) { + const clients = Object.values(this._virtualClients[server].userIds); + clients.forEach((bridgedClient) => { + if (bridgedClient.isDead()) { + return; // We don't want to include dead ones. + } + ageCounter.bump((Date.now() - bridgedClient.getLastActionTs()) / 1000); + }); +}; + +ClientPool.prototype.getNickUserIdMappingForChannel = function(server, channel) { + const nickUserIdMap = {}; + const cliSet = this._virtualClients[server.domain].userIds; + Object.keys(cliSet).filter((userId) => + cliSet[userId].chanList.includes(channel) + ).forEach((userId) => { + nickUserIdMap[cliSet[userId].nick] = userId; + }); + return nickUserIdMap; +}; ClientPool.prototype._sendConnectionMetric = function(server) { stats.ircClients(server.domain, this._getNumberOfConnections(server)); diff --git a/lib/irc/ConnectionInstance.js b/lib/irc/ConnectionInstance.js index abaa3fe95..a2dc5639a 100644 --- a/lib/irc/ConnectionInstance.js +++ b/lib/irc/ConnectionInstance.js @@ -28,6 +28,12 @@ const PING_RATE_MS = 1000 * 60; // String reply of any CTCP Version requests const CTCP_VERSION = 'matrix-appservice-irc, part of the Matrix.org Network'; +const CONN_LIMIT_MESSAGES = [ + "too many host connections", // ircd-seven + "no more connections allowed in your connection class", + "this server is full", // unrealircd +] + // Log an Error object to stderr function logError(err) { if (!err || !err.message) { @@ -93,7 +99,7 @@ ConnectionInstance.prototype.connect = function() { /** * Blow away the connection. You MUST destroy this object afterwards. * @param {string} reason - Reason to reject with. One of: - * throttled|irc_error|net_error|timeout|raw_error + * throttled|irc_error|net_error|timeout|raw_error|toomanyconns */ ConnectionInstance.prototype.disconnect = function(reason) { if (this.dead) { @@ -165,7 +171,7 @@ ConnectionInstance.prototype._listenForErrors = function() { "err_banonchan", "err_nickcollision", "err_nicknameinuse", "err_erroneusnickname", "err_nonicknamegiven", "err_eventnickchange", "err_nicktoofast", "err_unknowncommand", "err_unavailresource", - "err_umodeunknownflag" + "err_umodeunknownflag", "err_nononreg" ]; if (err && err.command) { if (failCodes.indexOf(err.command) !== -1) { @@ -215,6 +221,13 @@ ConnectionInstance.prototype._listenForErrors = function() { self.disconnect("banned").catch(logError); return; } + const tooManyHosts = CONN_LIMIT_MESSAGES.find((connLimitMsg) => { + return errText.includes(connLimitMsg); + }) !== undefined; + if (tooManyHosts) { + self.disconnect("toomanyconns").catch(logError); + return; + } } if (!wasThrottled) { self.disconnect("raw_error").catch(logError); @@ -317,6 +330,7 @@ ConnectionInstance.create = Promise.coroutine(function*(server, opts, onCreatedC retryCount: 0, family: server.getIpv6Prefix() || server.getIpv6Only() ? 6 : null, bustRfc3484: true, + sasl: opts.password ? server.useSasl() : false, }; if (server.useSsl()) { @@ -369,6 +383,13 @@ ConnectionInstance.create = Promise.coroutine(function*(server, opts, onCreatedC yield Promise.delay(BANNED_TIME_MS); } + if (err.message === "toomanyconns") { + log.error( + `User ${opts.nick} was ILINED. This may be the network limiting us!` + ); + throw new Error("Connection was ILINED. We cannot retry this."); + } + // always set a staggered delay here to avoid thundering herd // problems on mass-disconnects let delay = (BASE_RETRY_TIME_MS * Math.random())+ retryTimeMs + diff --git a/lib/irc/IrcEventBroker.js b/lib/irc/IrcEventBroker.js index baef33445..aa4d92688 100644 --- a/lib/irc/IrcEventBroker.js +++ b/lib/irc/IrcEventBroker.js @@ -347,6 +347,18 @@ IrcEventBroker.prototype.addHooks = function(client, connInst) { req, server, createUser(nick), chan, "join" )); }); + this._hookIfClaimed(client, connInst, "nick", function(oldNick, newNick, chans, msg) { + chans = chans || []; + chans.forEach((chan) => { + const req = createRequest(); + complete(req, ircHandler.onPart( + req, server, createUser(oldNick), chan, "nick" + )); + complete(req, ircHandler.onJoin( + req, server, createUser(newNick), chan, "nick" + )); + }); + }); // bucket names and drain them one at a time to avoid flooding // the matrix side with registrations / joins var namesBucket = [ diff --git a/lib/irc/IrcServer.js b/lib/irc/IrcServer.js index bd8074f44..88644b557 100644 --- a/lib/irc/IrcServer.js +++ b/lib/irc/IrcServer.js @@ -2,9 +2,10 @@ * Represents a single IRC server from config.yaml */ "use strict"; -var logging = require("../logging"); -var IrcClientConfig = require("../models/IrcClientConfig"); -var log = logging.get("IrcServer"); +const logging = require("../logging"); +const IrcClientConfig = require("../models/IrcClientConfig"); +const log = logging.get("IrcServer"); +const BridgedClient = require("./BridgedClient"); const GROUP_ID_REGEX = /^\+\S+:\S+$/; @@ -201,6 +202,10 @@ IrcServer.prototype.useSslSelfSigned = function() { return Boolean(this.config.sslselfsign); }; +IrcServer.prototype.useSasl = function() { + return Boolean(this.config.sasl); +}; + IrcServer.prototype.allowExpiredCerts = function() { return Boolean(this.config.allowExpiredCerts); }; @@ -429,10 +434,16 @@ IrcServer.prototype.getAliasFromChannel = function(channel) { }; IrcServer.prototype.getNick = function(userId, displayName) { - var localpart = userId.substring(1).split(":")[0]; - var display = displayName || localpart; - var template = this.config.ircClients.nickTemplate; - var nick = template.replace(/\$USERID/g, userId); + const illegalChars = BridgedClient.illegalCharactersRegex; + let localpart = userId.substring(1).split(":")[0]; + localpart = localpart.replace(illegalChars, ""); + displayName = displayName ? displayName.replace(illegalChars, "") : undefined; + const display = [displayName, localpart].find((n) => Boolean(n)); + if (!display) { + throw new Error("Could not get nick for user, all characters were invalid"); + } + const template = this.config.ircClients.nickTemplate; + let nick = template.replace(/\$USERID/g, userId); nick = nick.replace(/\$LOCALPART/g, localpart); nick = nick.replace(/\$DISPLAY/g, display); return nick; diff --git a/lib/irc/formatting.js b/lib/irc/formatting.js index dec71ce78..5319f5753 100644 --- a/lib/irc/formatting.js +++ b/lib/irc/formatting.js @@ -80,6 +80,9 @@ function escapeHtmlChars(text) { .replace(/'/g, "'"); // to work on HTML4 (' is HTML5 only) } +// It's useful! +module.exports.escapeHtmlChars = escapeHtmlChars; + /** * Given the state of the message, open or close an HTML tag with the * appropriate attributes. diff --git a/lib/models/MatrixAction.js b/lib/models/MatrixAction.js index 7e91ca51c..f839ca0d4 100644 --- a/lib/models/MatrixAction.js +++ b/lib/models/MatrixAction.js @@ -1,7 +1,10 @@ +/*eslint no-invalid-this: 0*/ // eslint doesn't understand Promise.coroutine wrapping "use strict"; -var ircFormatting = require("../irc/formatting"); -var log = require("../logging").get("MatrixAction"); -var ContentRepo = require("matrix-appservice-bridge").ContentRepo; +const ircFormatting = require("../irc/formatting"); +const log = require("../logging").get("MatrixAction"); +const ContentRepo = require("matrix-appservice-bridge").ContentRepo; +const escapeStringRegexp = require('escape-string-regexp'); +const Promise = require("bluebird"); const ACTION_TYPES = ["message", "emote", "topic", "notice", "file", "image", "video", "audio"]; const EVENT_TO_TYPE = { @@ -17,6 +20,9 @@ const MSGTYPE_TO_TYPE = { "m.file": "file" }; +const PILL_MIN_LENGTH_TO_MATCH = 4; +const MAX_MATCHES = 5; + function MatrixAction(type, text, htmlText, timestamp) { if (ACTION_TYPES.indexOf(type) === -1) { throw new Error("Unknown MatrixAction type: " + type); @@ -26,6 +32,67 @@ function MatrixAction(type, text, htmlText, timestamp) { this.htmlText = htmlText; this.ts = timestamp || 0; } + +MatrixAction.prototype.formatMentions = Promise.coroutine(function*(nickUserIdMap, intent) { + const regexString = "(" + + Object.keys(nickUserIdMap).map((value) => escapeStringRegexp(value)).join("|") + + ")"; + const usersRegex = MentionRegex(regexString); + const matched = new Set(); // lowercased nicknames we have matched already. + let match; + for (let i = 0; i < MAX_MATCHES && (match = usersRegex.exec(this.text)) !== null; i++) { + let matchName = match[2]; + // Deliberately have a minimum length to match on, + // so we don't match smaller nicks accidentally. + if (matchName.length < PILL_MIN_LENGTH_TO_MATCH || matched.has(matchName.toLowerCase())) { + continue; + } + let userId = nickUserIdMap[matchName]; + if (userId === undefined) { + // We might need to search case-insensitive. + const nick = Object.keys(nickUserIdMap).find((n) => + n.toLowerCase() === matchName.toLowerCase() + ); + if (nick === undefined) { + continue; + } + userId = nickUserIdMap[nick]; + matchName = nick; + } + // If this message is not HTML, we should make it so. + if (this.htmlText === undefined) { + // This looks scary and unsafe, but further down we check + // if `text` contains any HTML and escape + set `htmlText` appropriately. + this.htmlText = this.text; + } + userId = ircFormatting.escapeHtmlChars(userId); + + /* Due to how Riot and friends do push notifications, + we need the plain text to match something.*/ + let identifier; + try { + identifier = (yield intent.getProfileInfo(userId, 'displayname', true)).displayname; + } + catch (e) { + // This shouldn't happen, but let's not fail to match if so. + } + + if (identifier === undefined) { + // Fallback to userid. + identifier = userId.substr(1, userId.indexOf(":")-1) + } + + const regex = MentionRegex(escapeStringRegexp(matchName)); + this.htmlText = this.htmlText.replace(regex, + `$1`+ + `${ircFormatting.escapeHtmlChars(identifier)}` + ); + this.text = this.text.replace(regex, `$1${identifier}`); + // Don't match this name twice, we've already replaced all entries. + matched.add(matchName.toLowerCase()); + } +}); + MatrixAction.fromEvent = function(client, event, mediaUrl) { event.content = event.content || {}; let type = EVENT_TO_TYPE[event.type] || "message"; // mx event type to action type @@ -81,4 +148,12 @@ MatrixAction.fromIrcAction = function(ircAction) { } }; +function MentionRegex(matcher) { + const WORD_BOUNDARY = "^|\:|\#|```|\\s|$|,"; + return new RegExp( + `(${WORD_BOUNDARY})(@?(${matcher}))(?=${WORD_BOUNDARY})`, + "igmu" + ); +} + module.exports = MatrixAction; diff --git a/lib/provisioning/Provisioner.js b/lib/provisioning/Provisioner.js index 13fb356d5..1a8dfe18e 100644 --- a/lib/provisioning/Provisioner.js +++ b/lib/provisioning/Provisioner.js @@ -892,7 +892,10 @@ Provisioner.prototype.unlink = Promise.coroutine(function*(req) { Provisioner.prototype._leaveIfUnprovisioned = Promise.coroutine( function*(req, roomId, server, ircChannel) { try { - yield this._partUnlinkedIrcClients(req, roomId, server, ircChannel) + yield Promise.all([ + this._partUnlinkedIrcClients(req, roomId, server, ircChannel), + this._leaveMatrixVirtuals(req, roomId, server, ircChannel) + ]); } catch (err) { // keep going, we still need to part the bot; this is just cleanup @@ -935,9 +938,9 @@ Provisioner.prototype._partUnlinkedIrcClients = Promise.coroutine( // For each room, get the list of real matrix users and tally up how many times each one // appears as joined - let joinedUserCounts = Object.create(null); // user_id => Number + const joinedUserCounts = {}; // user_id => Number let unlinkedUserIds = []; - let asBot = this._ircBridge.getAppServiceBridge().getBot(); + const asBot = this._ircBridge.getAppServiceBridge().getBot(); for (let i = 0; i < matrixRooms.length; i++) { let stateEvents = []; try { @@ -947,7 +950,14 @@ Provisioner.prototype._partUnlinkedIrcClients = Promise.coroutine( req.log.error("Failed to hit /state for room " + matrixRooms[i].getId()); req.log.error(err.stack); } - let roomInfo = asBot._getRoomInfo(roomId, stateEvents); + + // _getRoomInfo takes a particular format. + const joinedRoom = { + state: { + events: stateEvents + } + } + let roomInfo = asBot._getRoomInfo(matrixRooms[i].getId(), joinedRoom); for (let j = 0; j < roomInfo.realJoinedUsers.length; j++) { let userId = roomInfo.realJoinedUsers[j]; if (!joinedUserCounts[userId]) { @@ -983,6 +993,31 @@ Provisioner.prototype._partUnlinkedIrcClients = Promise.coroutine( } ); +Provisioner.prototype._leaveMatrixVirtuals = Promise.coroutine( + function*(req, roomId, server, ircChannel) { + const asBot = this._ircBridge.getAppServiceBridge().getBot(); + const roomChannels = yield this._ircBridge.getStore().getIrcChannelsForRoomId( + roomId + ); + if (roomChannels.length > 0) { + // We can't determine who should and shouldn't be in the room. + return; + } + const stateEvents = yield asBot.getClient().roomState(roomId); + const roomInfo = asBot._getRoomInfo(roomId, { + state: { + events: stateEvents + } + }); + req.log.info(`Leaving ${roomInfo.remoteJoinedUsers.length} virtual users from ${roomId}.`); + this._ircBridge.memberListSyncers[server.domain].addToLeavePool( + roomInfo.remoteJoinedUsers, + roomId, + ircChannel + ); + } +); + // Cause the bot to leave the matrix room if there are no other channels being mapped to // this room Provisioner.prototype._leaveMatrixRoomIfUnprovisioned = Promise.coroutine( diff --git a/lib/util/QueuePool.js b/lib/util/QueuePool.js index 7d100370b..1c54735ad 100644 --- a/lib/util/QueuePool.js +++ b/lib/util/QueuePool.js @@ -28,7 +28,7 @@ class QueuePool { } // Get number of items waiting to be inserted into a queue. - get waitingItems() { return this._overflowCount; } + get waitingItems() { return this.overflow.size(); } // Add an item to the queue. ID and item are passed directly to the Queue. // Index is optional and should be between 0 ~ poolSize-1. It determines @@ -54,7 +54,6 @@ class QueuePool { // onto the queue pool. We want to return a promise which resolves // after the item has finished executing on the queue pool, hence // the promise chain here. - this._overflowCount++; return this.overflow.enqueue(id, { id: id, item: item, @@ -78,7 +77,6 @@ class QueuePool { return q.onceFree(); }); return Promise.any(promises).then((q) => { - this._overflowCount--; if (q.size() !== 0) { throw new Error(`QueuePool overflow: starvation. No free queues.`); } diff --git a/package.json b/package.json index e984e230a..a600c3714 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-appservice-irc", - "version": "0.10.1", + "version": "0.11.0", "description": "An IRC Bridge for Matrix", "main": "app.js", "bin": "./bin/matrix-appservice-irc", @@ -11,7 +11,7 @@ "test": "BLUEBIRD_DEBUG=1 jasmine --stop-on-failure=true", "lint": "eslint --max-warnings 0 lib spec", "check": "npm test && npm run lint", - "ci-test": "BLUEBIRD_DEBUG=1 istanbul cover -x \"**/spec/**\" --report text jasmine", + "ci-test": "BLUEBIRD_DEBUG=1 nyc --report text jasmine", "ci": "npm run lint && npm run ci-test" }, "repository": { @@ -26,10 +26,11 @@ "dependencies": { "bluebird": "^3.1.1", "crc": "^3.2.1", + "escape-string-regexp": "^1.0.5", "extend": "^2.0.0", - "irc": "matrix-org/node-irc#b1614bc784200c65247784d7b9e9ab867140412d", + "irc": "matrix-org/node-irc#c9abb427bec5016d94a2abf3e058cc62de09ea5a", "js-yaml": "^3.2.7", - "matrix-appservice-bridge": "1.5.0a", + "matrix-appservice-bridge": "1.6.0c", "nedb": "^1.1.2", "nopt": "^3.0.1", "prom-client": "^6.3.0", @@ -39,9 +40,9 @@ "winston-daily-rotate-file": "^3.2.1" }, "devDependencies": { + "jasmine": "^3.1.0", + "nyc": "^12.0.2", "eslint": "^4.15.0", - "istanbul": "^0.4.5", - "jasmine": "^2.5.2", "proxyquire": "^1.4.0" } } diff --git a/spec/integ/matrix-to-irc.spec.js b/spec/integ/matrix-to-irc.spec.js index 3234e3953..c0531cb61 100644 --- a/spec/integ/matrix-to-irc.spec.js +++ b/spec/integ/matrix-to-irc.spec.js @@ -240,6 +240,200 @@ describe("Matrix-to-IRC message bridging", function() { }); }); + it("should bridge matrix replies as roughly formatted text", function(done) { + // Trigger an original event + env.mockAppService._trigger("type:m.room.message", { + content: { + body: "This is the real message", + msgtype: "m.text" + }, + user_id: testUser.id, + room_id: roomMapping.roomId, + sender: "@friend:bar.com", + event_id: "$original:bar.com", + type: "m.room.message" + }).then(() => { + env.ircMock._whenClient(roomMapping.server, testUser.nick, "say", + function(client, channel, text) { + expect(client.nick).toEqual(testUser.nick); + expect(client.addr).toEqual(roomMapping.server); + expect(channel).toEqual(roomMapping.channel); + expect(text).toEqual(' Reply Text'); + done(); + }); + const formatted_body = constructHTMLReply( + "This is the fake message", + "@somedude:bar.com", + "Reply text" + ); + env.mockAppService._trigger("type:m.room.message", { + content: { + body: "> <@somedude:bar.com> This is the fake message\n\nReply Text", + formatted_body, + format: "org.matrix.custom.html", + msgtype: "m.text", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$original:bar.com" + } + }, + }, + user_id: testUser.id, + room_id: roomMapping.roomId, + type: "m.room.message" + }); + }); + }); + + it("should bridge matrix replies as roughly formatted text, newline edition", function(done) { + // Trigger an original event + env.mockAppService._trigger("type:m.room.message", { + content: { + body: "\nThis\n is the real message", + msgtype: "m.text" + }, + user_id: testUser.id, + room_id: roomMapping.roomId, + sender: "@friend:bar.com", + event_id: "$original:bar.com", + type: "m.room.message" + }).then(() => { + env.ircMock._whenClient(roomMapping.server, testUser.nick, "say", + function(client, channel, text) { + expect(client.nick).toEqual(testUser.nick); + expect(client.addr).toEqual(roomMapping.server); + expect(channel).toEqual(roomMapping.channel); + expect(text).toEqual(' Reply Text'); + done(); + }); + const formatted_body = constructHTMLReply( + "This is the fake message", + "@somedude:bar.com", + "Reply text" + ); + env.mockAppService._trigger("type:m.room.message", { + content: { + body: "> <@somedude:bar.com> This is the fake message\n\nReply Text", + formatted_body, + format: "org.matrix.custom.html", + msgtype: "m.text", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$original:bar.com" + } + }, + }, + user_id: testUser.id, + room_id: roomMapping.roomId, + type: "m.room.message" + }); + }); + }); + + it("should bridge matrix replies as reply only, if source not found", function(done) { + env.ircMock._whenClient(roomMapping.server, testUser.nick, "say", + function(client, channel, text) { + expect(client.nick).toEqual(testUser.nick); + expect(client.addr).toEqual(roomMapping.server); + expect(channel).toEqual(roomMapping.channel); + expect(text).toEqual('Reply Text'); + done(); + }); + const formatted_body = constructHTMLReply( + "This message is possibly fake", + "@somedude:bar.com", + "Reply Text" + ); + + env.mockAppService._trigger("type:m.room.message", { + content: { + body: "> <@somedude:bar.com> This message is possibly fake\n\nReply Text", + msgtype: "m.text", + formatted_body, + format: "org.matrix.custom.html", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$original:bar.com" + } + }, + }, + formatted_body, + user_id: testUser.id, + room_id: roomMapping.roomId, + type: "m.room.message" + }); + }); + + it("should bridge matrix replies to replies without the original source", function(done) { + let formatted_body; + env.mockAppService._trigger("type:m.room.message", { + content: { + body: "Message #1", + msgtype: "m.text" + }, + user_id: testUser.id, + room_id: roomMapping.roomId, + sender: "@friend:bar.com", + event_id: "$first:bar.com", + type: "m.room.message" + }).then(() => { + formatted_body = constructHTMLReply( + "Message #1", + "@somedude:bar.com", + "Message #2" + ); + return env.mockAppService._trigger("type:m.room.message", { + content: { + body: "> <@friend:bar.com> Message#1\n\nMessage #2", + formatted_body, + format: "org.matrix.custom.html", + msgtype: "m.text", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$first:bar.com" + } + }, + }, + user_id: testUser.id, + room_id: roomMapping.roomId, + sender: "@friend:bar.com", + event_id: "$second:bar.com", + type: "m.room.message" + }); + }).then(() => { + formatted_body = constructHTMLReply( + "Message #2", + "@somedude:bar.com", + "Message #3" + ); + env.ircMock._whenClient(roomMapping.server, testUser.nick, "say", + function(client, channel, text) { + expect(client.nick).toEqual(testUser.nick); + expect(client.addr).toEqual(roomMapping.server); + expect(channel).toEqual(roomMapping.channel); + expect(text).toEqual(' Message #3'); + done(); + }); + + env.mockAppService._trigger("type:m.room.message", { + content: { + body: "> <@friend:bar.com> Message#2\n\nMessage #3", + formatted_body, + format: "org.matrix.custom.html", + msgtype: "m.text", + "m.relates_to": { + "m.in_reply_to": { + "event_id": "$second:bar.com" + } + }, + }, + user_id: testUser.id, + room_id: roomMapping.roomId, + type: "m.room.message" + }); + }); + }); + it("should bridge matrix images as IRC action with a URL", function(done) { var tBody = "the_image.jpg"; var tMxcSegment = "/somecontentid"; @@ -656,3 +850,10 @@ describe("Matrix-to-IRC message bridging with media URL and drop time", function }); }); }); + +function constructHTMLReply(sourceText, sourceUser, reply) { + // This is one hella ugly format. + return "
In reply to" + + `${sourceUser}

${sourceText}

${reply}`; +} diff --git a/spec/unit/IrcServer.spec.js b/spec/unit/IrcServer.spec.js new file mode 100644 index 000000000..ab359b3ae --- /dev/null +++ b/spec/unit/IrcServer.spec.js @@ -0,0 +1,56 @@ +"use strict"; +const IrcServer = require("../../lib/irc/IrcServer"); +const extend = require("extend"); +describe("IrcServer", function() { + describe("getNick", function() { + it("should get a nick from a userid", function() { + const server = new IrcServer("irc.foobar", + extend(true, IrcServer.DEFAULT_CONFIG, {}) + ); + expect(server.getNick("@foobar:foobar")).toBe("M-foobar"); + }); + it("should get a nick from a displayname", function() { + const server = new IrcServer("irc.foobar", + extend(true, IrcServer.DEFAULT_CONFIG, {}) + ); + expect(server.getNick("@foobar:foobar", "wiggle")).toBe("M-wiggle"); + }); + it("should get a reduced nick if the displayname contains some invalid chars", function() { + const server = new IrcServer("irc.foobar", + extend(true, IrcServer.DEFAULT_CONFIG, {}) + ); + expect(server.getNick("@foobar:foobar", "💩wiggleケ")).toBe("M-wiggle"); + }); + it("should use localpart if the displayname is all invalid chars", function() { + const server = new IrcServer("irc.foobar", + extend(true, IrcServer.DEFAULT_CONFIG, {}) + ); + expect(server.getNick("@foobar:foobar", "💩ケ")).toBe("M-foobar"); + }); + // These situations shouldn't happen, but we want to avoid rogue homeservers blowing us up. + it("should get a reduced nick if the localpart contains some invalid chars", function() { + const server = new IrcServer("irc.foobar", + extend(true, IrcServer.DEFAULT_CONFIG, {}) + ); + expect(server.getNick("@💩foobarケ:foobar")).toBe("M-foobar"); + }); + it("should use displayname if the localpart is all invalid chars", function() { + const server = new IrcServer("irc.foobar", + extend(true, IrcServer.DEFAULT_CONFIG, {}) + ); + expect(server.getNick("@💩ケ:foobar", "wiggle")).toBe("M-wiggle"); + }); + it("should throw if no characters could be used, with displayname", function() { + const server = new IrcServer("irc.foobar", + extend(true, IrcServer.DEFAULT_CONFIG, {}) + ); + expect(() => {server.getNick("@💩ケ:foobar", "💩ケ")}).toThrow(); + }); + it("should throw if no characters could be used, with displayname", function() { + const server = new IrcServer("irc.foobar", + extend(true, IrcServer.DEFAULT_CONFIG, {}) + ); + expect(() => {server.getNick("@💩ケ:foobar")}).toThrow(); + }); + }); +}); diff --git a/spec/unit/MatrixAction.spec.js b/spec/unit/MatrixAction.spec.js new file mode 100644 index 000000000..51b5fe453 --- /dev/null +++ b/spec/unit/MatrixAction.spec.js @@ -0,0 +1,176 @@ +"use strict"; +const MatrixAction = require("../../lib/models/MatrixAction"); + +const FakeIntent = { + getProfileInfo: (userId) => { + return new Promise((resolve, reject) => { + if (userId === "@jc.denton:unatco.gov") { + resolve({displayname: "TheJCDenton"}); + } + else if (userId === "@paul.denton:unatco.gov") { + resolve({displayname: "ThePaulDenton"}); + } + else { + reject("This user was not found"); + } + }); + } +} + +describe("MatrixAction", function() { + + it("should not highlight mentions to text without mentions", () => { + let action = new MatrixAction("message", "Some text"); + return action.formatMentions({ + "Some Person": "@foobar:localhost" + }, FakeIntent).then(() => { + expect(action.text).toEqual("Some text"); + expect(action.htmlText).toBeUndefined(); + }); + }); + + it("should highlight a user", () => { + let action = new MatrixAction( + "message", + "JCDenton, it's a bomb!", + "JCDenton, it's a bomb!", + null + ); + return action.formatMentions({ + "JCDenton": "@jc.denton:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("TheJCDenton, it's a bomb!"); + expect(action.htmlText).toEqual( + ""+ + "TheJCDenton, it's a bomb!" + ); + }); + }); + it("should highlight a user, regardless of case", () => { + let action = new MatrixAction( + "message", + "JCDenton, it's a bomb!", + "JCDenton, it's a bomb!", + null + ); + return action.formatMentions({ + "jcdenton": "@jc.denton:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("TheJCDenton, it's a bomb!"); + expect(action.htmlText).toEqual( + ""+ + "TheJCDenton, it's a bomb!" + ); + }); + + }); + it("should highlight a user, with plain text", () => { + let action = new MatrixAction("message", "JCDenton, it's a bomb!"); + return action.formatMentions({ + "JCDenton": "@jc.denton:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("TheJCDenton, it's a bomb!"); + expect(action.htmlText).toEqual( + ""+ + "TheJCDenton, it's a bomb!" + ); + }); + }); + it("should highlight a user, with weird characters", () => { + let action = new MatrixAction("message", "`||JCDenton[m], it's a bomb!"); + return action.formatMentions({ + "`||JCDenton[m]": "@jc.denton:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("TheJCDenton, it's a bomb!"); + expect(action.htmlText).toEqual( + ""+ + "TheJCDenton, it's a bomb!" + ); + }); + }); + it("should highlight multiple users", () => { + let action = new MatrixAction( + "message", + "JCDenton is sent to assassinate PaulDenton", + "JCDenton is sent to assassinate PaulDenton", + null + ); + return action.formatMentions({ + "JCDenton": "@jc.denton:unatco.gov", + "PaulDenton": "@paul.denton:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("TheJCDenton is sent to assassinate ThePaulDenton"); + expect(action.htmlText).toEqual( + "TheJCDenton is sent" + + " to assassinate " + + "ThePaulDenton" + ); + }); + }); + it("should highlight multiple mentions of the same user", () => { + let action = new MatrixAction( + "message", + "JCDenton, meet JCDenton", + "JCDenton, meet JCDenton", + null + ); + return action.formatMentions({ + "JCDenton": "@jc.denton:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("TheJCDenton, meet TheJCDenton"); + expect(action.htmlText).toEqual( + "TheJCDenton," + + " meet TheJCDenton" + ); + }); + }); + it("should not highlight mentions in a URL with www.", () => { + let action = new MatrixAction( + "message", + "Go to http://www.JCDenton.com", + "Go to my website", + null + ); + return action.formatMentions({ + "JCDenton": "@jc.denton:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("Go to http://www.JCDenton.com"); + expect(action.htmlText).toEqual( + "Go to my website" + ); + }); + }); + it("should not highlight mentions in a URL with http://", () => { + let action = new MatrixAction( + "message", + "Go to http://JCDenton.com", + "Go to my website", + null + ); + return action.formatMentions({ + "JCDenton": "@jc.denton:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("Go to http://JCDenton.com"); + expect(action.htmlText).toEqual( + "Go to my website" + ); + }); + }); + it("should fallback to userIds", () => { + let action = new MatrixAction( + "message", + "AnnaNavarre: The machine would not make a mistake!", + "AnnaNavarre: The machine would not make a mistake!", + null + ); + return action.formatMentions({ + "AnnaNavarre": "@anna.navarre:unatco.gov" + }, FakeIntent).then(() => { + expect(action.text).toEqual("anna.navarre: The machine would not make a mistake!"); + expect(action.htmlText).toEqual( + ""+ + "anna.navarre: The machine would not make a mistake!" + ); + }); + }); +}); diff --git a/spec/unit/Queue.spec.js b/spec/unit/Queue.spec.js index 80b35ed0f..a55b8a371 100644 --- a/spec/unit/Queue.spec.js +++ b/spec/unit/Queue.spec.js @@ -115,4 +115,25 @@ describe("Queue", function() { done(); }); }); + + it("should have the correct size", (done) => { + const thing1 = { foo: "bar"}; + const thing2 = { bar: "baz"}; + const things = [thing1, thing2]; + let expectedSize = things.length; + procFn.and.callFake((thing) => { + things.shift(); + expect(queue.size()).toEqual(expectedSize); + if (things.length === 0) { + done(); + } + expectedSize--; + return Promise.resolve(); + }); + expect(queue.size()).toEqual(0); + queue.enqueue("id1", thing1); + expect(queue.size()).toEqual(1); + queue.enqueue("id2", thing2); + expect(queue.size()).toEqual(2); + }); }); diff --git a/spec/unit/QueuePool.spec.js b/spec/unit/QueuePool.spec.js index 23cac283e..9a5e38c3f 100644 --- a/spec/unit/QueuePool.spec.js +++ b/spec/unit/QueuePool.spec.js @@ -34,7 +34,7 @@ describe("QueuePool", function() { procFn.and.callFake((item) => { itemToDeferMap[item] = new promiseutil.defer(); return itemToDeferMap[item].promise; - }) + }); }); it("should let multiple items be processed at once", @@ -135,4 +135,16 @@ describe("QueuePool", function() { yield nextTick(); expect(Object.keys(itemToDeferMap).sort()).toEqual(["b"]); })); + + it("should accurately track waiting items", test.coroutine(function*() { + for (let i = 0; i < 10; i++) { + pool.enqueue(i, i); + } + expect(pool.waitingItems).toEqual(7); + for (let j = 0; j < 10; j++) { + yield nextTick(); + resolveItem(j); + } + expect(pool.waitingItems).toEqual(0); + })); }); diff --git a/spec/util/client-sdk-mock.js b/spec/util/client-sdk-mock.js index 99871649b..ca6a2cde8 100644 --- a/spec/util/client-sdk-mock.js +++ b/spec/util/client-sdk-mock.js @@ -13,6 +13,7 @@ function MockClient(config) { userId: config.userId }; this._http = { opts: {} }; + this._http.authedRequest = jasmine.createSpy("sdk.authedRequestWithPrefix()"); this._http.authedRequestWithPrefix = jasmine.createSpy("sdk.authedRequestWithPrefix()"); this.register = jasmine.createSpy("sdk.register(username, password)"); this.createRoom = jasmine.createSpy("sdk.createRoom(opts)"); @@ -39,6 +40,10 @@ function MockClient(config) { return Promise.resolve({}); }); + this._http.authedRequest.and.callFake(function() { + return Promise.resolve({}); + }); + this._http.authedRequestWithPrefix.and.callFake(function(a, method, endpoint) { if (endpoint === "/joined_rooms") { return Promise.resolve([]);