v10.0.0

Restrict event emit for room members that had power levels changed

.babelrc

@@ -1,15 +1,19 @@
 {
-    "presets": ["es2015"],
-    "plugins": [
-        "transform-class-properties",
-
-        // this transforms async functions into generator functions, which
-        // are then made to use the regenerator module by babel's
-        // transform-regnerator plugin (which is enabled by es2015).
-        "transform-async-to-bluebird",
-
-        // This makes sure that the regenerator runtime is available to
-        // the transpiled code.
-        "transform-runtime",
+    "sourceMaps": true,
+    "presets": [
+        ["@babel/preset-env", {
+            "targets": {
+                "node": 10
+            },
+            "modules": "commonjs"
+        }],
+        "@babel/preset-typescript"
     ],
+    "plugins": [
+        "@babel/plugin-proposal-numeric-separator",
+        "@babel/plugin-proposal-class-properties",
+        "@babel/plugin-proposal-object-rest-spread",
+        "@babel/plugin-syntax-dynamic-import",
+        "@babel/plugin-transform-runtime"
+    ]
 }

.buildkite/pipeline.yaml

@@ -1,34 +0,0 @@
-steps:
-  - label: ":eslint: Lint"
-    command:
-      - "yarn install"
-      - "yarn lint"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:10"
-
-  - label: ":karma: Tests"
-    command:
-      - "yarn install"
-      - "yarn test"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:10"
-
-  - label: "📃 Docs"
-    command:
-      - "yarn install"
-      - "yarn gendoc"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:10"
-
-  - wait
-
-  - label: "🐴 Trigger matrix-react-sdk"
-    trigger: "matrix-react-sdk"
-    branches: "develop"
-    build:
-        branch: "develop"
-        message: "[js-sdk] ${BUILDKITE_MESSAGE}"
-    async: true

.eslintrc.js

@@ -1,69 +1,14 @@
 module.exports = {
-    parser: "babel-eslint", // now needed for class properties
-    parserOptions: {
-        sourceType: "module",
-        ecmaFeatures: {
-        }
-    },
+    plugins: [
+        "matrix-org",
+    ],
+    extends: ["plugin:matrix-org/javascript"],
     env: {
         browser: true,
         node: true,
-
-        // babel's transform-runtime converts references to ES6 globals such as
-        // Promise and Map to core-js polyfills, so we can use ES6 globals.
-        es6: true,
     },
-    extends: ["eslint:recommended", "google"],
-    plugins: [
-        "babel",
-    ],
     rules: {
-        // rules we've always adhered to or now do
-        "max-len": ["error", {
-            code: 90,
-            ignoreComments: true,
-        }],
-        curly: ["error", "multi-line"],
-        "prefer-const": ["error"],
-        "comma-dangle": ["error", {
-            arrays: "always-multiline",
-            objects: "always-multiline",
-            imports: "always-multiline",
-            exports: "always-multiline",
-            functions: "always-multiline",
-        }],
-
-        // loosen jsdoc requirements a little
-        "require-jsdoc": ["error", {
-            require: {
-                FunctionDeclaration: false,
-            }
-        }],
-        "valid-jsdoc": ["error", {
-            requireParamDescription: false,
-            requireReturn: false,
-            requireReturnDescription: false,
-        }],
-
-        // rules we do not want from eslint-recommended
-        "no-console": ["off"],
-        "no-constant-condition": ["off"],
-        "no-empty": ["error", { "allowEmptyCatch": true }],
-
-        // rules we do not want from the google styleguide
-        "object-curly-spacing": ["off"],
-        "spaced-comment": ["off"],
-        "guard-for-in": ["off"],
-
-        // in principle we prefer single quotes, but life is too short
-        quotes: ["off"],
-
-        // rules we'd ideally like to adhere to, but the current
-        // code does not (in most cases because it's still ES5)
-        // we set these to warnings, and assert that the number
-        // of warnings doesn't exceed a given threshold
         "no-var": ["warn"],
-        "brace-style": ["warn", "1tbs", {"allowSingleLine": true}],
         "prefer-rest-params": ["warn"],
         "prefer-spread": ["warn"],
         "one-var": ["warn"],
@@ -77,10 +22,23 @@ module.exports = {
             "asyncArrow": "always",
         }],
         "arrow-parens": "off",
-
-        // eslint's built in no-invalid-this rule breaks with class properties
-        "no-invalid-this": "off",
-        // so we replace it with a version that is class property aware
-        "babel/no-invalid-this": "error",
-    }
-}
+        "prefer-promise-reject-errors": "off",
+        "quotes": "off",
+        "indent": "off",
+        "no-constant-condition": "off",
+        "no-async-promise-executor": "off",
+        // We use a `logger` intermediary module
+        "no-console": "error",
+    },
+    overrides: [{
+        "files": ["src/**/*.ts"],
+        "extends": ["plugin:matrix-org/typescript"],
+        "rules": {
+            // We're okay being explicit at the moment
+            "@typescript-eslint/no-empty-interface": "off",
+            // While we're converting to ts we make heavy use of this
+            "@typescript-eslint/no-explicit-any": "off",
+            "quotes": "off",
+        },
+    }],
+};

.gitignore

@@ -10,10 +10,8 @@ build/Release
 coverage
 lib-cov
 out
-reports
 /dist
 /lib
-/specbuild
 
 # version file and tarball created by `npm pack` / `yarn pack`
 /git-revision.txt

CONTRIBUTING.rst

@@ -28,16 +28,23 @@ use GitHub's pull request workflow to review the contribution, and either ask
 you to make any refinements needed or merge it and make them ourselves. The
 changes will then land on master when we next do a release.
 
-We use Travis for continuous integration, and all pull requests get
-automatically tested by Travis: if your change breaks the build, then the PR
-will show that there are failed checks, so please check back after a few
-minutes.
+We use continuous integration, and all pull requests get automatically tested:
+if your change breaks the build, then the PR will show that there are failed
+checks, so please check back after a few minutes.
 
 Code style
 ~~~~~~~~~~
 
-The code-style for matrix-js-sdk is not formally documented, but contributors
-are encouraged to read the code style document for matrix-react-sdk
+The js-sdk aims to target TypeScript/ES6. All new files should be written in
+TypeScript and existing files should use ES6 principles where possible.
+
+Members should not be exported as a default export in general - it causes problems
+with the architecture of the SDK (index file becomes less clear) and could
+introduce naming problems (as default exports get aliased upon import). In
+general, avoid using `export default`.
+
+The remaining code-style for matrix-js-sdk is not formally documented, but
+contributors are encouraged to read the code style document for matrix-react-sdk
 (`<https://github.com/matrix-org/matrix-react-sdk/blob/master/code_style.md>`_)
 and follow the principles set out there.
 

README.md

@@ -9,12 +9,16 @@ Quickstart
 
 In a browser
 ------------
-Download either the full or minified version from
+Download the browser version from
 https://github.com/matrix-org/matrix-js-sdk/releases/latest and add that as a
 ``<script>`` to your page. There will be a global variable ``matrixcs``
 attached to ``window`` through which you can access the SDK. See below for how to
 include libolm to enable end-to-end-encryption.
 
+The browser bundle supports recent versions of browsers. Typically this is ES2015
+or `> 0.5%, last 2 versions, Firefox ESR, not dead` if using 
+[browserlists](https://github.com/browserslist/browserslist).
+
 Please check [the working browser example](examples/browser) for more information.
 
 In Node.js
@@ -22,13 +26,18 @@ In Node.js
 
 Ensure you have the latest LTS version of Node.js installed.
 
-Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://yarnpkg.com/docs/install/) if you do not have it already.
+This SDK targets Node 10 for compatibility, which translates to ES6. If you're using
+a bundler like webpack you'll likely have to transpile dependencies, including this
+SDK, to match your target browsers.
+
+Using `yarn` instead of `npm` is recommended. Please see the Yarn [install guide](https://classic.yarnpkg.com/en/docs/install) 
+if you do not have it already.
 
 ``yarn add matrix-js-sdk``
 
 ```javascript
-  var sdk = require("matrix-js-sdk");
-  var client = sdk.createClient("https://matrix.org");
+  import * as sdk from "matrix-js-sdk";
+  const client = sdk.createClient("https://matrix.org");
   client.publicRooms(function(err, data) {
     console.log("Public Rooms: %s", JSON.stringify(data));
   });
@@ -59,7 +68,7 @@ client.once('sync', function(state, prevState, res) {
 To send a message:
 
 ```javascript
-var content = {
+const content = {
     "body": "message text",
     "msgtype": "m.text"
 };
@@ -161,7 +170,7 @@ which will be fulfilled in the future.
 The typical usage is something like:
 
 ```javascript
-  matrixClient.someMethod(arg1, arg2).done(function(result) {
+  matrixClient.someMethod(arg1, arg2).then(function(result) {
     ...
   });
 ```
@@ -173,10 +182,8 @@ you can pass the result of the promise into it with something like:
   matrixClient.someMethod(arg1, arg2).nodeify(callback);
 ```
 
-The main thing to note is that it is an error to discard the result of a
-promise-returning function, as that will cause exceptions to go unobserved. If
-you have nothing better to do with the result, just call ``.done()`` on it. See
-http://documentup.com/kriskowal/q/#the-end for more information.
+The main thing to note is that it is problematic to discard the result of a
+promise-returning function, as that will cause exceptions to go unobserved.
 
 Methods which return a promise show this in their documentation.
 
@@ -191,10 +198,10 @@ This section provides some useful code snippets which demonstrate the
 core functionality of the SDK. These examples assume the SDK is setup like this:
 
 ```javascript
-   var sdk = require("matrix-js-sdk");
-   var myUserId = "@example:localhost";
-   var myAccessToken = "QGV4YW1wbGU6bG9jYWxob3N0.qPEvLuYfNBjxikiCjP";
-   var matrixClient = sdk.createClient({
+   import * as sdk from "matrix-js-sdk";
+   const myUserId = "@example:localhost";
+   const myAccessToken = "QGV4YW1wbGU6bG9jYWxob3N0.qPEvLuYfNBjxikiCjP";
+   const matrixClient = sdk.createClient({
        baseUrl: "http://localhost:8008",
        accessToken: myAccessToken,
        userId: myUserId
@@ -206,7 +213,7 @@ core functionality of the SDK. These examples assume the SDK is setup like this:
 ```javascript
    matrixClient.on("RoomMember.membership", function(event, member) {
        if (member.membership === "invite" && member.userId === myUserId) {
-           matrixClient.joinRoom(member.roomId).done(function() {
+           matrixClient.joinRoom(member.roomId).then(function() {
                console.log("Auto-joined %s", member.roomId);
            });
        }
@@ -247,11 +254,11 @@ Output:
 
 ```javascript
    matrixClient.on("RoomState.members", function(event, state, member) {
-       var room = matrixClient.getRoom(state.roomId);
+       const room = matrixClient.getRoom(state.roomId);
        if (!room) {
            return;
        }
-       var memberList = state.getMembers();
+       const memberList = state.getMembers();
        console.log(room.name);
        console.log(Array(room.name.length + 1).join("="));  // underline
        for (var i = 0; i < memberList.length; i++) {
@@ -297,10 +304,10 @@ End-to-end encryption support
 =============================
 
 The SDK supports end-to-end encryption via the Olm and Megolm protocols, using
-[libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the 
+[libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the
 application to make libolm available, via the ``Olm`` global.
 
-It is also necessry to call ``matrixClient.initCrypto()`` after creating a new
+It is also necessary to call ``matrixClient.initCrypto()`` after creating a new
 ``MatrixClient`` (but **before** calling ``matrixClient.startClient()``) to
 initialise the crypto layer.
 
@@ -319,16 +326,16 @@ To provide the Olm library in a browser application:
 
  * download the transpiled libolm (from https://packages.matrix.org/npm/olm/).
  * load ``olm.js`` as a ``<script>`` *before* ``browser-matrix.js``.
- 
+
 To provide the Olm library in a node.js application:
 
- * ``yarn add https://packages.matrix.org/npm/olm/olm-3.0.0.tgz``
+ * ``yarn add https://packages.matrix.org/npm/olm/olm-3.1.4.tgz``
    (replace the URL with the latest version you want to use from
     https://packages.matrix.org/npm/olm/)
  * ``global.Olm = require('olm');`` *before* loading ``matrix-js-sdk``.
 
 If you want to package Olm as dependency for your node.js application, you can
-use ``yarn add https://packages.matrix.org/npm/olm/olm-3.0.0.tgz``. If your
+use ``yarn add https://packages.matrix.org/npm/olm/olm-3.1.4.tgz``. If your
 application also works without e2e crypto enabled, add ``--optional`` to mark it
 as an optional dependency.
 
@@ -351,11 +358,6 @@ To build a browser version from scratch when developing::
  $ yarn build
 ```
 
-To constantly do builds when files are modified (using ``watchify``)::
-```
- $ yarn watch
-```
-
 To run tests (Jasmine)::
 ```
  $ yarn test

browser-index.js

@@ -1,35 +0,0 @@
-var matrixcs = require("./lib/matrix");
-const request = require('browser-request');
-const queryString = require('qs');
-
-matrixcs.request(function(opts, fn) {
-    // We manually fix the query string for browser-request because
-    // it doesn't correctly handle cases like ?via=one&via=two. Instead
-    // we mimic `request`'s query string interface to make it all work
-    // as expected.
-    // browser-request will happily take the constructed string as the
-    // query string without trying to modify it further.
-    opts.qs = queryString.stringify(opts.qs || {}, opts.qsStringifyOptions);
-    return request(opts, fn);
-});
-
-// just *accessing* indexedDB throws an exception in firefox with
-// indexeddb disabled.
-var indexedDB;
-try {
-    indexedDB = global.indexedDB;
-} catch(e) {}
-
-// if our browser (appears to) support indexeddb, use an indexeddb crypto store.
-if (indexedDB) {
-    matrixcs.setCryptoStoreFactory(
-        function() {
-            return new matrixcs.IndexedDBCryptoStore(
-                indexedDB, "matrix-js-sdk:crypto"
-            );
-        }
-    );
-}
-
-module.exports = matrixcs; // keep export for browserify package deps
-global.matrixcs = matrixcs;

examples/browser/browserTest.js

@@ -1,4 +1,3 @@
-"use strict";
 console.log("Loading browser sdk");
 
 var client = matrixcs.createClient("http://matrix.org");

examples/crypto-browser/lib/.gitignore

@@ -0,0 +1,2 @@
+olm.js
+olm.wasm

examples/crypto-browser/lib/matrix.js

@@ -0,0 +1 @@
+../../../dist/browser-matrix.js

examples/crypto-browser/olm-device-export-import.html

@@ -0,0 +1,59 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>Test Crypto in Browser</title>
+    <script src="lib/olm.js"></script>
+    <script src="lib/matrix.js"></script>
+</head>
+<body>
+    <h1>Testing export/import of Olm devices in the browser</h1>
+    <ul>
+        <li>
+            Make sure you built the current version of the Matrix JS SDK
+            (<code>yarn build</code>)
+        </li>
+        <li>
+            copy <code>olm.js</code> and <code>olm.wasm</code>
+            from a recent release of Olm (was tested with version 3.1.4)
+            in directory <code>lib/</code>
+        </li>
+        <li>start a local Matrix homeserver (on port 8008, or change the port in the code)</li>
+        <li>Serve this HTML file (e.g. <code>python3 -m http.server</code>) and go to it through your browser</li>
+        <li>
+            in the JS console, do:
+            <pre>
+aliceMatrixClient = await newMatrixClient("alice-"+randomHex());
+await aliceMatrixClient.exportDevice();
+await aliceMatrixClient.getAccessToken();
+            </pre>
+        </li>
+        <li>
+            copy the result of <code>exportDevice</code> and <code>getAccessToken</code> somewhere
+            (<strong>not</strong> in a JS variable as it will be destroyed when you refresh the page)
+        </li>
+        <li><strong>refresh the page (F5)</strong> to make sure the client is destroyed</li>
+        <li>
+            Do the following, replacing <code>ALICE_ID</code>
+            with the user ID of Alice (you can find it in the exported data)
+            <pre>
+bobMatrixClient = await newMatrixClient("bob-"+randomHex());
+roomId = await bobMatrixClient.createEncryptedRoom([ALICE_ID]);
+await bobMatrixClient.sendTextMessage('Hi Alice!', roomId);
+            </pre>
+        </li>
+        <li>Again, <strong>refresh the page (F5)</strong>. You may want to clear your console as well.</li>
+        <li>
+            Now do the following, using the exported data and the access token you saved previously:
+            <pre>
+aliceMatrixClient = await importMatrixClient(EXPORTED_DATA, ACCESS_TOKEN);
+            </pre>
+        </li>
+        <li>You should see the message sent by Bob printed in the console.</li>
+    </ul>
+
+    <script src="olm-device-export-import.js"></script>
+</body>
+</html>

examples/crypto-browser/olm-device-export-import.js

@@ -0,0 +1,122 @@
+if (!Olm) {
+    console.error(
+        "global.Olm does not seem to be present."
+        + " Did you forget to add olm in the lib/ directory?"
+    );
+}
+
+const BASE_URL = 'http://localhost:8008';
+const ROOM_CRYPTO_CONFIG = { algorithm: 'm.megolm.v1.aes-sha2' };
+const PASSWORD = 'password';
+
+// useful to create new usernames
+window.randomHex = () => Math.floor(Math.random() * (10**6)).toString(16);
+
+window.newMatrixClient = async function (username) {
+    const registrationClient = matrixcs.createClient(BASE_URL);
+
+    const userRegisterResult = await registrationClient.register(
+      username,
+      PASSWORD,
+      null,
+      { type: 'm.login.dummy' }
+    );
+  
+    const matrixClient = matrixcs.createClient({
+      baseUrl: BASE_URL,
+      userId: userRegisterResult.user_id,
+      accessToken: userRegisterResult.access_token,
+      deviceId: userRegisterResult.device_id,
+      sessionStore: new matrixcs.WebStorageSessionStore(window.localStorage),
+      cryptoStore: new matrixcs.MemoryCryptoStore(),
+    });
+
+    extendMatrixClient(matrixClient);
+
+    await matrixClient.initCrypto();
+    await matrixClient.startClient();
+    return matrixClient;
+}
+
+window.importMatrixClient = async function (exportedDevice, accessToken) {
+    const matrixClient = matrixcs.createClient({
+      baseUrl: BASE_URL,
+      deviceToImport: exportedDevice,
+      accessToken,
+      sessionStore: new matrixcs.WebStorageSessionStore(window.localStorage),
+      cryptoStore: new matrixcs.MemoryCryptoStore(),
+    });
+
+    extendMatrixClient(matrixClient);
+
+    await matrixClient.initCrypto();
+    await matrixClient.startClient();
+    return matrixClient;
+}
+
+function extendMatrixClient(matrixClient) {
+    // automatic join
+    matrixClient.on('RoomMember.membership', async (event, member) => {
+        if (member.membership === 'invite' && member.userId === matrixClient.getUserId()) {
+            await matrixClient.joinRoom(member.roomId);
+            // setting up of room encryption seems to be triggered automatically
+            // but if we don't wait for it the first messages we send are unencrypted
+            await matrixClient.setRoomEncryption(member.roomId, { algorithm: 'm.megolm.v1.aes-sha2' })
+        }
+    });
+
+    matrixClient.onDecryptedMessage = message => {
+        console.log('Got encrypted message: ', message);
+    }
+
+    matrixClient.on('Event.decrypted', (event) => {
+        if (event.getType() === 'm.room.message'){
+            matrixClient.onDecryptedMessage(event.getContent().body);
+        } else {
+            console.log('decrypted an event of type', event.getType());
+            console.log(event);
+        }
+    });
+  
+    matrixClient.createEncryptedRoom = async function(usersToInvite) {
+        const {
+          room_id: roomId,
+        } = await this.createRoom({
+          visibility: 'private',
+          invite: usersToInvite,
+        });
+
+        // matrixClient.setRoomEncryption() only updates local state
+        // but does not send anything to the server
+        // (see https://github.com/matrix-org/matrix-js-sdk/issues/905)
+        // so we do it ourselves with 'sendStateEvent'
+        await this.sendStateEvent(
+          roomId, 'm.room.encryption', ROOM_CRYPTO_CONFIG,
+        );
+        await this.setRoomEncryption(
+          roomId, ROOM_CRYPTO_CONFIG,
+        );
+
+        // Marking all devices as verified
+        let room = this.getRoom(roomId);
+        let members = (await room.getEncryptionTargetMembers()).map(x => x["userId"])
+        let memberkeys = await this.downloadKeys(members);
+        for (const userId in memberkeys) {
+          for (const deviceId in memberkeys[userId]) {
+            await this.setDeviceVerified(userId, deviceId);
+          }
+        }
+
+        return roomId;
+    }
+
+    matrixClient.sendTextMessage = async function(message, roomId) {
+        return matrixClient.sendMessage(
+            roomId,
+            {
+                body: message,
+                msgtype: 'm.text',
+            }
+        )
+    }
+}

examples/node/app.js

@@ -1,5 +1,3 @@
-"use strict";
-
 var myUserId = "@example:localhost";
 var myAccessToken = "QGV4YW1wbGU6bG9jYWxob3N0.qPEvLuYfNBjxikiCjP";
 var sdk = require("matrix-js-sdk");
@@ -56,7 +54,7 @@ rl.on('line', function(line) {
                 }
             }
             if (notSentEvent) {
-                matrixClient.resendEvent(notSentEvent, viewingRoom).done(function() {
+                matrixClient.resendEvent(notSentEvent, viewingRoom).then(function() {
                     printMessages();
                     rl.prompt();
                 }, function(err) {
@@ -70,7 +68,7 @@ rl.on('line', function(line) {
         }
         else if (line.indexOf("/more ") === 0) {
             var amount = parseInt(line.split(" ")[1]) || 20;
-            matrixClient.scrollback(viewingRoom, amount).done(function(room) {
+            matrixClient.scrollback(viewingRoom, amount).then(function(room) {
                 printMessages();
                 rl.prompt();
             }, function(err) {
@@ -79,7 +77,7 @@ rl.on('line', function(line) {
         }
         else if (line.indexOf("/invite ") === 0) {
             var userId = line.split(" ")[1].trim();
-            matrixClient.invite(viewingRoom.roomId, userId).done(function() {
+            matrixClient.invite(viewingRoom.roomId, userId).then(function() {
                 printMessages();
                 rl.prompt();
             }, function(err) {
@@ -92,7 +90,7 @@ rl.on('line', function(line) {
             matrixClient.uploadContent({
                 stream: stream,
                 name: filename
-            }).done(function(url) {
+            }).then(function(url) {
                 var content = {
                     msgtype: "m.file",
                     body: filename,
@@ -116,7 +114,7 @@ rl.on('line', function(line) {
             viewingRoom = roomList[roomIndex];
             if (viewingRoom.getMember(myUserId).membership === "invite") {
                 // join the room first
-                matrixClient.joinRoom(viewingRoom.roomId).done(function(room) {
+                matrixClient.joinRoom(viewingRoom.roomId).then(function(room) {
                     setRoomList();
                     viewingRoom = room;
                     printMessages();
@@ -128,7 +126,7 @@ rl.on('line', function(line) {
             else {
                 printMessages();
             }
-        } 
+        }
     }
     rl.prompt();
 });
@@ -281,8 +279,8 @@ function printMemberList(room) {
             member.membership + new Array(10 - member.membership.length).join(" ")
         );
         print(
-            "%s"+fmt(" :: ")+"%s"+fmt(" (")+"%s"+fmt(")"), 
-            membershipWithPadding, member.name, 
+            "%s"+fmt(" :: ")+"%s"+fmt(" (")+"%s"+fmt(")"),
+            membershipWithPadding, member.name,
             (member.userId === myUserId ? "Me" : member.userId),
             fmt
         );
@@ -290,26 +288,27 @@ function printMemberList(room) {
 }
 
 function printRoomInfo(room) {
-    var eventDict = room.currentState.events;
+    var eventMap = room.currentState.events;
     var eTypeHeader = "    Event Type(state_key)    ";
     var sendHeader = "        Sender        ";
     // pad content to 100
     var restCount = (
-        100 - "Content".length - " | ".length - " | ".length - 
+        100 - "Content".length - " | ".length - " | ".length -
         eTypeHeader.length - sendHeader.length
     );
     var padSide = new Array(Math.floor(restCount/2)).join(" ");
     var contentHeader = padSide + "Content" + padSide;
     print(eTypeHeader+sendHeader+contentHeader);
     print(new Array(100).join("-"));
-    Object.keys(eventDict).forEach(function(eventType) {
+    eventMap.keys().forEach(function(eventType) {
         if (eventType === "m.room.member") { return; } // use /members instead.
-        Object.keys(eventDict[eventType]).forEach(function(stateKey) {
+        var eventEventMap = eventMap.get(eventType);
+        eventEventMap.keys().forEach(function(stateKey) {
             var typeAndKey = eventType + (
                 stateKey.length > 0 ? "("+stateKey+")" : ""
             );
             var typeStr = fixWidth(typeAndKey, eTypeHeader.length);
-            var event = eventDict[eventType][stateKey];
+            var event = eventEventMap.get(stateKey);
             var sendStr = fixWidth(event.getSender(), sendHeader.length);
             var contentStr = fixWidth(
                 JSON.stringify(event.getContent()), contentHeader.length

examples/voip/browserTest.js

@@ -1,4 +1,3 @@
-"use strict";
 console.log("Loading browser sdk");
 var BASE_URL = "https://matrix.org";
 var TOKEN = "accesstokengoeshere";

index.js

@@ -1,6 +0,0 @@
-var matrixcs = require("./lib/matrix");
-matrixcs.request(require("request"));
-module.exports = matrixcs;
-
-var utils = require("./lib/utils");
-utils.runPolyfills();

jenkins.sh

@@ -1,36 +0,0 @@
-#!/bin/bash -l
-
-set -x
-
-export NVM_DIR="$HOME/.nvm"
-[ -s "$NVM_DIR/nvm.sh" ] && . "$NVM_DIR/nvm.sh"
-
-nvm use 10 || exit $?
-yarn install || exit $?
-
-RC=0
-
-function fail {
-    echo $@ >&2
-    RC=1
-}
-
-# don't use last time's test reports
-rm -rf reports coverage || exit $?
-
-yarn test || fail "yarn test finished with return code $?"
-
-yarn -s lint -f checkstyle > eslint.xml ||
-    fail "eslint finished with return code $?"
-
-# delete the old tarball, if it exists
-rm -f matrix-js-sdk-*.tgz
-
-# `yarn pack` doesn't seem to run scripts, however that seems okay here as we
-# just built as part of `install` above.
-yarn pack ||
-    fail "yarn pack finished with return code $?"
-
-yarn gendoc || fail "JSDoc failed with code $?"
-
-exit $RC

jsdoc.json

@@ -0,0 +1,30 @@
+{
+  "tags": {
+    "allowUnknownTags": true
+  },
+  "plugins": [
+    "node_modules/better-docs/category",
+    "node_modules/better-docs/typescript"
+  ],
+  "source": {
+    "include": [
+      "src"
+    ],
+    "includePattern": ".(ts|js)$"
+  },
+  "opts": {
+    "encoding": "utf8",
+    "destination": ".jsdoc",
+    "readme": "README.md",
+    "recurse": true,
+    "verbose": true,
+    "template": "node_modules/docdash"
+  },
+  "docdash": {
+    "static": true,
+    "private": false,
+    "search": true,
+    "collapse": true,
+    "typedefs": true
+  }
+}

package.json

@@ -1,24 +1,24 @@
 {
   "name": "matrix-js-sdk",
-  "version": "2.3.2",
+  "version": "10.0.0",
   "description": "Matrix Client-Server SDK for Javascript",
-  "main": "index.js",
   "scripts": {
-    "test:build": "babel -s -d specbuild spec",
-    "test:run": "istanbul cover --report text --report cobertura --config .istanbul.yml -i \"lib/**/*.js\" node_modules/mocha/bin/_mocha -- --recursive specbuild --colors --reporter mocha-jenkins-reporter --reporter-options junit_report_path=reports/test-results.xml",
-    "test:watch": "mocha --watch --compilers js:babel-core/register --recursive spec --colors",
-    "test": "yarn test:build && yarn test:run",
-    "check": "yarn test:build && _mocha --recursive specbuild --colors",
-    "gendoc": "babel --no-babelrc --plugins transform-class-properties -d .jsdocbuild src && jsdoc -r .jsdocbuild -P package.json -R README.md -d .jsdoc",
-    "start": "yarn start:init && yarn start:watch",
-    "start:watch": "babel -s -w --skip-initial-build -d lib src",
-    "start:init": "babel -s -d lib src",
+    "prepublishOnly": "yarn build",
+    "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"",
+    "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build",
     "clean": "rimraf lib dist",
-    "build": "babel -s -d lib src && rimraf dist && mkdir dist && browserify -d browser-index.js | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js && terser -c -m -o dist/browser-matrix.min.js --source-map \"content='dist/browser-matrix.js.map'\" dist/browser-matrix.js",
-    "dist": "yarn build",
-    "watch": "watchify -d browser-index.js -o 'exorcist dist/browser-matrix.js.map > dist/browser-matrix.js' -v",
-    "lint": "eslint --max-warnings 101 src spec",
-    "prepare": "yarn clean && yarn build && git rev-parse HEAD > git-revision.txt"
+    "build": "yarn build:dev && yarn build:compile-browser && yarn build:minify-browser",
+    "build:dev": "yarn clean && git rev-parse HEAD > git-revision.txt && yarn build:compile && yarn build:types",
+    "build:types": "tsc --emitDeclarationOnly",
+    "build:compile": "babel -d lib --verbose --extensions \".ts,.js\" src",
+    "build:compile-browser": "mkdirp dist && browserify -d src/browser-index.js -p [ tsify -p ./tsconfig.json ] -t [ babelify --sourceMaps=inline --presets [ @babel/preset-env @babel/preset-typescript ] ] | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js",
+    "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
+    "gendoc": "jsdoc -c jsdoc.json -P package.json",
+    "lint": "yarn lint:types && yarn lint:js",
+    "lint:js": "eslint --max-warnings 72 src spec",
+    "lint:types": "tsc --noEmit",
+    "test": "jest spec/ --coverage --testEnvironment node",
+    "test:watch": "jest spec/ --coverage --testEnvironment node --watch"
   },
   "repository": {
     "type": "git",
@@ -27,72 +27,77 @@
   "keywords": [
     "matrix-org"
   ],
-  "browser": "browser-index.js",
+  "main": "./lib/index.js",
+  "browser": "./lib/browser-index.js",
+  "matrix_src_main": "./src/index.ts",
+  "matrix_src_browser": "./src/browser-index.js",
+  "matrix_lib_main": "./lib/index.js",
+  "matrix_lib_typings": "./lib/index.d.ts",
   "author": "matrix.org",
   "license": "Apache-2.0",
   "files": [
-    ".babelrc",
-    ".eslintrc.js",
-    "spec/.eslintrc.js",
+    "dist",
+    "lib",
+    "src",
+    "git-revision.txt",
     "CHANGELOG.md",
     "CONTRIBUTING.rst",
     "LICENSE",
     "README.md",
-    "RELEASING.md",
-    "examples",
-    "git-hooks",
-    "git-revision.txt",
-    "index.js",
-    "browser-index.js",
-    "jenkins.sh",
-    "lib",
     "package.json",
-    "release.sh",
-    "spec",
-    "src"
+    "release.sh"
   ],
   "dependencies": {
+    "@babel/runtime": "^7.12.5",
     "another-json": "^0.2.0",
-    "babel-runtime": "^6.26.0",
-    "bluebird": "^3.5.0",
     "browser-request": "^0.3.3",
     "bs58": "^4.0.1",
-    "content-type": "^1.0.2",
-    "loglevel": "1.6.1",
-    "qs": "^6.5.2",
-    "request": "^2.88.0",
-    "unhomoglyph": "^1.0.2"
+    "content-type": "^1.0.4",
+    "loglevel": "^1.7.1",
+    "qs": "^6.9.6",
+    "request": "^2.88.2",
+    "unhomoglyph": "^1.0.6"
   },
   "devDependencies": {
-    "babel-cli": "^6.18.0",
-    "babel-eslint": "^10.0.1",
-    "babel-plugin-transform-async-to-bluebird": "^1.1.1",
-    "babel-plugin-transform-class-properties": "^6.24.1",
-    "babel-plugin-transform-runtime": "^6.23.0",
-    "babel-preset-es2015": "^6.18.0",
-    "browserify": "^16.2.3",
-    "browserify-shim": "^3.8.13",
-    "eslint": "^5.12.0",
-    "eslint-config-google": "^0.7.1",
-    "eslint-plugin-babel": "^5.3.0",
-    "exorcist": "^0.4.0",
-    "expect": "^1.20.2",
-    "istanbul": "^0.4.5",
-    "jsdoc": "^3.5.5",
-    "lolex": "^1.5.2",
+    "@babel/cli": "^7.12.10",
+    "@babel/core": "^7.12.10",
+    "@babel/eslint-parser": "^7.12.10",
+    "@babel/eslint-plugin": "^7.12.10",
+    "@babel/plugin-proposal-class-properties": "^7.12.1",
+    "@babel/plugin-proposal-numeric-separator": "^7.12.7",
+    "@babel/plugin-proposal-object-rest-spread": "^7.12.1",
+    "@babel/plugin-syntax-dynamic-import": "^7.8.3",
+    "@babel/plugin-transform-runtime": "^7.12.10",
+    "@babel/preset-env": "^7.12.11",
+    "@babel/preset-typescript": "^7.12.7",
+    "@babel/register": "^7.12.10",
+    "@types/jest": "^26.0.20",
+    "@types/node": "12",
+    "@types/request": "^2.48.5",
+    "@typescript-eslint/eslint-plugin": "^4.17.0",
+    "@typescript-eslint/parser": "^4.17.0",
+    "babel-jest": "^26.6.3",
+    "babelify": "^10.0.0",
+    "better-docs": "^2.3.2",
+    "browserify": "^17.0.0",
+    "docdash": "^1.2.0",
+    "eslint": "7.18.0",
+    "eslint-config-google": "^0.14.0",
+    "eslint-plugin-matrix-org": "github:matrix-org/eslint-plugin-matrix-org#main",
+    "exorcist": "^1.0.1",
+    "fake-indexeddb": "^3.1.2",
+    "jest": "^26.6.3",
+    "jest-localstorage-mock": "^2.4.6",
+    "jsdoc": "^3.6.6",
     "matrix-mock-request": "^1.2.3",
-    "mocha": "^5.2.0",
-    "mocha-jenkins-reporter": "^0.4.0",
-    "olm": "https://packages.matrix.org/npm/olm/olm-3.1.0.tgz",
-    "rimraf": "^2.5.4",
-    "source-map-support": "^0.4.11",
-    "sourceify": "^0.1.0",
-    "terser": "^4.0.0",
-    "watchify": "^3.11.1"
+    "olm": "https://packages.matrix.org/npm/olm/olm-3.2.1.tgz",
+    "rimraf": "^3.0.2",
+    "terser": "^5.5.1",
+    "tsify": "^5.0.2",
+    "typescript": "^4.1.3"
   },
-  "browserify": {
-    "transform": [
-      "sourceify"
-    ]
-  }
+  "jest": {
+    "testEnvironment": "node"
+  },
+  "typings": "./lib/index.d.ts"
 }

release.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to perform a release of matrix-js-sdk.
+# Script to perform a release of matrix-js-sdk and downstream projects.
 #
 # Requires:
 #   github-changelog-generator; install via:
@@ -9,6 +9,8 @@
 #   hub; install via brew (macOS) or source/pre-compiled binaries (debian) (https://github.com/github/hub) - Tested on v2.2.9
 #   npm; typically installed by Node.js
 #   yarn; install via brew (macOS) or similar (https://yarnpkg.com/docs/install/)
+#
+# Note: this script is also used to release matrix-react-sdk and element-web.
 
 set -e
 
@@ -36,6 +38,7 @@ $USAGE
     -c changelog_file:  specify name of file containing changelog
     -x:                 skip updating the changelog
     -z:                 skip generating the jsdoc
+    -n:                 skip publish to NPM
 EOF
 }
 
@@ -58,9 +61,10 @@ fi
 
 skip_changelog=
 skip_jsdoc=
+skip_npm=
 changelog_file="CHANGELOG.md"
 expected_npm_user="matrixdotorg"
-while getopts hc:u:xz f; do
+while getopts hc:u:xzn f; do
     case $f in
         h)
             help
@@ -75,6 +79,9 @@ while getopts hc:u:xz f; do
         z)
             skip_jsdoc=1
             ;;
+        n)
+            skip_npm=1
+            ;;
         u)
             expected_npm_user="$OPTARG"
             ;;
@@ -87,6 +94,14 @@ if [ $# -ne 1 ]; then
     exit 1
 fi
 
+# We use Git branch / commit dependencies for some packages, and Yarn seems
+# to have a hard time getting that right. See also
+# https://github.com/yarnpkg/yarn/issues/4734. As a workaround, we clean the
+# global cache here to ensure we get the right thing.
+yarn cache clean
+# Ensure all dependencies are updated
+yarn install --ignore-scripts
+
 if [ -z "$skip_changelog" ]; then
     # update_changelog doesn't have a --version flag
     update_changelog -h > /dev/null || (echo "github-changelog-generator is required: please install it"; exit)
@@ -94,10 +109,12 @@ fi
 
 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-actual_npm_user=`npm whoami`;
-if [ $expected_npm_user != $actual_npm_user ]; then
-    echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
-    exit 1
+if [ -z "$skip_npm" ]; then
+    actual_npm_user=`npm whoami`;
+    if [ $expected_npm_user != $actual_npm_user ]; then
+        echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+        exit 1
+    fi
 fi
 
 # ignore leading v on release
@@ -161,6 +178,19 @@ echo "yarn version"
 # manually commit the result.
 yarn version --no-git-tag-version --new-version "$release"
 
+# For the published and dist versions of the package, we copy the
+# `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if
+# they exist). This small bit of gymnastics allows us to use the TypeScript
+# source directly for development without needing to build before linting or
+# testing.
+for i in main typings
+do
+    lib_value=$(jq -r ".matrix_lib_$i" package.json)
+    if [ "$lib_value" != "null" ]; then
+        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json
+    fi
+done
+
 # commit yarn.lock if it exists, is versioned, and is modified
 if [[ -f yarn.lock && `git status --porcelain yarn.lock | grep '^ M'` ]];
 then
@@ -289,7 +319,16 @@ rm "${latest_changes}"
 
 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-npm publish
+# Tag both releases and prereleases as `next` so the last stable release remains
+# the default.
+if [ -z "$skip_npm" ]; then
+    npm publish --tag next
+    if [ $prerelease -eq 0 ]; then
+        # For a release, also add the default `latest` tag.
+        package=$(cat package.json | jq -er .name)
+        npm dist-tag add "$package@$release" latest
+    fi
+fi
 
 if [ -z "$skip_jsdoc" ]; then
     echo "generating jsdocs"
@@ -304,6 +343,7 @@ if [ -z "$skip_jsdoc" ]; then
         $release index.html
     git add "$release"
     git commit --no-verify -m "Add jsdoc for $release" index.html "$release"
+    git push origin gh-pages
 fi
 
 # if it is a pre-release, leave it on the release branch for now.
@@ -316,16 +356,40 @@ fi
 echo "updating master branch"
 git checkout master
 git pull
-git merge "$rel_branch"
+git merge "$rel_branch" --no-edit
 
-# push master and docs (if generated) to github
+# push master to github
 git push origin master
-if [ -z "$skip_jsdoc" ]; then
-    git push origin gh-pages
-fi
 
-# finally, merge master back onto develop
-git checkout develop
-git pull
-git merge master
-git push origin develop
+# finally, merge master back onto develop (if it exists)
+if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
+    git checkout develop
+    git pull
+    git merge master --no-edit
+
+    # When merging to develop, we need revert the `main` and `typings` fields if
+    # we adjusted them previously.
+    for i in main typings
+    do
+        # If a `lib` prefixed value is present, it means we adjusted the field
+        # earlier at publish time, so we should revert it now.
+        if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
+            # If there's a `src` prefixed value, use that, otherwise delete.
+            # This is used to delete the `typings` field and reset `main` back
+            # to the TypeScript source.
+            src_value=$(jq -r ".matrix_src_$i" package.json)
+            if [ "$src_value" != "null" ]; then
+                jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json
+            else
+                jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json
+            fi
+        fi
+    done
+
+    if [ -n "$(git ls-files --modified package.json)" ]; then
+        echo "Committing develop package.json"
+        git commit package.json -m "Resetting package fields for development"
+    fi
+
+    git push origin develop
+fi

spec/.eslintrc.js

@@ -1,5 +0,0 @@
-module.exports = {
-    env: {
-        mocha: true,
-    },
-}

spec/MockStorageApi.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -18,7 +19,7 @@ limitations under the License.
  * A mock implementation of the webstorage api
  * @constructor
  */
-function MockStorageApi() {
+export function MockStorageApi() {
     this.data = {};
     this.keys = [];
     this.length = 0;
@@ -52,5 +53,3 @@ MockStorageApi.prototype = {
     },
 };
 
-/** */
-module.exports = MockStorageApi;

spec/TestClient.js

@@ -16,18 +16,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-"use strict";
-
 // load olm before the sdk if possible
 import './olm-loader';
 
-import sdk from '..';
-import testUtils from './test-utils';
 import MockHttpBackend from 'matrix-mock-request';
-import expect from 'expect';
-import Promise from 'bluebird';
-import LocalStorageCryptoStore from '../lib/crypto/store/localStorage-crypto-store';
-import logger from '../src/logger';
+import {LocalStorageCryptoStore} from '../src/crypto/store/localStorage-crypto-store';
+import {logger} from '../src/logger';
+import {WebStorageSessionStore} from "../src/store/session/webstorage";
+import {syncPromise} from "./test-utils";
+import {createClient} from "../src/matrix";
+import {MockStorageApi} from "./MockStorageApi";
 
 /**
  * Wrapper for a MockStorageApi, MockHttpBackend and MatrixClient
@@ -41,16 +39,16 @@ import logger from '../src/logger';
  *     session store. If undefined, we will create a MockStorageApi.
  * @param {object} options additional options to pass to the client
  */
-export default function TestClient(
+export function TestClient(
     userId, deviceId, accessToken, sessionStoreBackend, options,
 ) {
     this.userId = userId;
     this.deviceId = deviceId;
 
     if (sessionStoreBackend === undefined) {
-        sessionStoreBackend = new testUtils.MockStorageApi();
+        sessionStoreBackend = new MockStorageApi();
     }
-    const sessionStore = new sdk.WebStorageSessionStore(sessionStoreBackend);
+    const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
 
     this.httpBackend = new MockHttpBackend();
 
@@ -67,10 +65,13 @@ export default function TestClient(
         this.cryptoStore = new LocalStorageCryptoStore(sessionStoreBackend);
         options.cryptoStore = this.cryptoStore;
     }
-    this.client = sdk.createClient(options);
+    this.client = createClient(options);
 
     this.deviceKeys = null;
     this.oneTimeKeys = {};
+    this._callEventHandler = {
+        calls: new Map(),
+    };
 }
 
 TestClient.prototype.toString = function() {
@@ -99,7 +100,7 @@ TestClient.prototype.start = function() {
 
     return Promise.all([
         this.httpBackend.flushAllExpected(),
-        testUtils.syncPromise(this.client),
+        syncPromise(this.client),
     ]).then(() => {
         logger.log(this + ': started');
     });
@@ -159,7 +160,7 @@ TestClient.prototype.awaitOneTimeKeyUpload = function() {
           .respond(200, (path, content) => {
               expect(content.device_keys).toBe(undefined);
               expect(content.one_time_keys).toBeTruthy();
-              expect(content.one_time_keys).toNotEqual({});
+              expect(content.one_time_keys).not.toEqual({});
               logger.log('%s: received %i one-time keys', this,
                           Object.keys(content.one_time_keys).length);
               this.oneTimeKeys = content.one_time_keys;
@@ -187,7 +188,7 @@ TestClient.prototype.expectKeyQuery = function(response) {
         200, (path, content) => {
             Object.keys(response.device_keys).forEach((userId) => {
                 expect(content.device_keys[userId]).toEqual(
-                    {},
+                    [],
                     "Expected key query for " + userId + ", got " +
                     Object.keys(content.device_keys),
                 );
@@ -227,8 +228,12 @@ TestClient.prototype.flushSync = function() {
     logger.log(`${this}: flushSync`);
     return Promise.all([
         this.httpBackend.flush('/sync', 1),
-        testUtils.syncPromise(this.client),
+        syncPromise(this.client),
     ]).then(() => {
         logger.log(`${this}: flushSync completed`);
     });
 };
+
+TestClient.prototype.isFallbackICEServerAllowed = function() {
+    return true;
+};

spec/browserify/setupTests.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,13 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-export function randomString(len) {
-    let ret = "";
-    const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+// stub for browser-matrix browserify tests
+global.XMLHttpRequest = jest.fn();
 
-    for (let i = 0; i < len; ++i) {
-        ret += chars.charAt(Math.floor(Math.random() * chars.length));
-    }
-
-    return ret;
-}
+afterAll(() => {
+    // clean up XMLHttpRequest mock
+    global.XMLHttpRequest = undefined;
+});

spec/browserify/sync-browserify.spec.js

@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// load XmlHttpRequest mock
+import "./setupTests";
+import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
+import {MockStorageApi} from "../MockStorageApi";
+import {WebStorageSessionStore} from "../../src/store/session/webstorage";
+import MockHttpBackend from "matrix-mock-request";
+import {LocalStorageCryptoStore} from "../../src/crypto/store/localStorage-crypto-store";
+import * as utils from "../test-utils";
+
+const USER_ID = "@user:test.server";
+const DEVICE_ID = "device_id";
+const ACCESS_TOKEN = "access_token";
+const ROOM_ID = "!room_id:server.test";
+
+/* global matrixcs */
+
+describe("Browserify Test", function() {
+    let client;
+    let httpBackend;
+
+    async function createTestClient() {
+        const sessionStoreBackend = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
+        const httpBackend = new MockHttpBackend();
+
+        const options = {
+            baseUrl: "http://" + USER_ID + ".test.server",
+            userId: USER_ID,
+            accessToken: ACCESS_TOKEN,
+            deviceId: DEVICE_ID,
+            sessionStore: sessionStore,
+            request: httpBackend.requestFn,
+            cryptoStore: new LocalStorageCryptoStore(sessionStoreBackend),
+        };
+
+        const client = matrixcs.createClient(options);
+
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+
+        return { client, httpBackend };
+    }
+
+    beforeEach(async () => {
+        ({client, httpBackend} = await createTestClient());
+        await client.startClient();
+    });
+
+    afterEach(async () => {
+        client.stopClient();
+        await httpBackend.stop();
+    });
+
+    it("Sync", async function() {
+        const event = utils.mkMembership({
+            room: ROOM_ID,
+            mship: "join",
+            user: "@other_user:server.test",
+            name: "Displayname",
+        });
+
+        const syncData = {
+            next_batch: "batch1",
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [
+                    event,
+                ],
+                limited: false,
+            },
+        };
+
+        httpBackend.when("GET", "/sync").respond(200, syncData);
+        await Promise.race([
+            Promise.all([
+                httpBackend.flushAllExpected(),
+            ]),
+            new Promise((_, reject) => {
+                client.once("sync.unexpectedError", reject);
+            }),
+        ]);
+    }, 20000); // additional timeout as this test can take quite a while
+});

spec/integ/devicelist-integ-spec.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,12 +16,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import expect from 'expect';
-import Promise from 'bluebird';
-
-import TestClient from '../TestClient';
-import testUtils from '../test-utils';
-import logger from '../../src/logger';
+import {TestClient} from '../TestClient';
+import * as testUtils from '../test-utils';
+import {logger} from '../../src/logger';
 
 const ROOM_ID = "!room:id";
 
@@ -88,8 +86,6 @@ describe("DeviceList management:", function() {
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-
         // we create our own sessionStoreBackend so that we can use it for
         // another TestClient.
         sessionStoreBackend = new testUtils.MockStorageApi();
@@ -144,7 +140,7 @@ describe("DeviceList management:", function() {
 
     it("We should not get confused by out-of-order device query responses",
        () => {
-           // https://github.com/vector-im/riot-web/issues/3126
+           // https://github.com/vector-im/element-web/issues/3126
            aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
            return aliceTestClient.start().then(() => {
                aliceTestClient.httpBackend.when('GET', '/sync').respond(
@@ -275,7 +271,7 @@ describe("DeviceList management:", function() {
            });
        }).timeout(3000);
 
-    // https://github.com/vector-im/riot-web/issues/4983
+    // https://github.com/vector-im/element-web/issues/4983
     describe("Alice should know she has stale device lists", () => {
         beforeEach(async function() {
             await aliceTestClient.start();

spec/integ/matrix-client-crypto.spec.js

@@ -2,6 +2,7 @@
 Copyright 2016 OpenMarket Ltd
 Copyright 2017 Vector Creations Ltd
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,19 +25,14 @@ limitations under the License.
  * See also `megolm.spec.js`.
  */
 
-"use strict";
-import 'source-map-support/register';
-
 // load olm before the sdk if possible
 import '../olm-loader';
 
-import expect from 'expect';
-const sdk = require("../..");
-import Promise from 'bluebird';
-const utils = require("../../lib/utils");
-const testUtils = require("../test-utils");
-const TestClient = require('../TestClient').default;
-import logger from '../../src/logger';
+import {logger} from '../../src/logger';
+import * as testUtils from "../test-utils";
+import * as utils from "../../src/utils";
+import {TestClient} from "../TestClient";
+import {CRYPTO_ENABLED} from "../../src/client";
 
 let aliTestClient;
 const roomId = "!room:localhost";
@@ -56,7 +52,7 @@ function bobUploadsDeviceKeys() {
         bobTestClient.client.uploadKeys(),
         bobTestClient.httpBackend.flush(),
     ]).then(() => {
-        expect(Object.keys(bobTestClient.deviceKeys).length).toNotEqual(0);
+        expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
     });
 }
 
@@ -74,7 +70,7 @@ function expectAliQueryKeys() {
     aliTestClient.httpBackend.when("POST", "/keys/query")
             .respond(200, function(path, content) {
         expect(content.device_keys[bobUserId]).toEqual(
-            {},
+            [],
             "Expected Alice to key query for " + bobUserId + ", got " +
             Object.keys(content.device_keys),
         );
@@ -102,7 +98,7 @@ function expectBobQueryKeys() {
         "POST", "/keys/query",
     ).respond(200, function(path, content) {
         expect(content.device_keys[aliUserId]).toEqual(
-            {},
+            [],
             "Expected Bob to key query for " + aliUserId + ", got " +
             Object.keys(content.device_keys),
         );
@@ -204,7 +200,7 @@ function aliSendsFirstMessage() {
         expectAliQueryKeys()
             .then(expectAliClaimKeys)
             .then(expectAliSendMessageRequest),
-    ]).spread(function(_, ciphertext) {
+    ]).then(function([_, ciphertext]) {
         return ciphertext;
     });
 }
@@ -219,7 +215,7 @@ function aliSendsMessage() {
     return Promise.all([
         sendMessage(aliTestClient.client),
         expectAliSendMessageRequest(),
-    ]).spread(function(_, ciphertext) {
+    ]).then(function([_, ciphertext]) {
         return ciphertext;
     });
 }
@@ -235,7 +231,7 @@ function bobSendsReplyMessage() {
         sendMessage(bobTestClient.client),
         expectBobQueryKeys()
             .then(expectBobSendMessageRequest),
-    ]).spread(function(_, ciphertext) {
+    ]).then(function([_, ciphertext]) {
         return ciphertext;
     });
 }
@@ -280,16 +276,17 @@ function sendMessage(client) {
 
 function expectSendMessageRequest(httpBackend) {
     const path = "/send/m.room.encrypted/";
-    const deferred = Promise.defer();
-    httpBackend.when("PUT", path).respond(200, function(path, content) {
-        deferred.resolve(content);
-        return {
-            event_id: "asdfgh",
-        };
+    const prom = new Promise((resolve) => {
+        httpBackend.when("PUT", path).respond(200, function(path, content) {
+            resolve(content);
+            return {
+                event_id: "asdfgh",
+            };
+        });
     });
 
     // it can take a while to process the key query
-    return httpBackend.flush(path, 1).then(() => deferred.promise);
+    return httpBackend.flush(path, 1).then(() => prom);
 }
 
 function aliRecvMessage() {
@@ -401,13 +398,11 @@ function firstSync(testClient) {
 
 
 describe("MatrixClient crypto", function() {
-    if (!sdk.CRYPTO_ENABLED) {
+    if (!CRYPTO_ENABLED) {
         return;
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-
         aliTestClient = new TestClient(aliUserId, aliDeviceId, aliAccessToken);
         await aliTestClient.client.initCrypto();
 
@@ -430,15 +425,14 @@ describe("MatrixClient crypto", function() {
             .then(bobUploadsDeviceKeys);
     });
 
-    it("Ali downloads Bobs device keys", function(done) {
-        Promise.resolve()
+    it("Ali downloads Bobs device keys", function() {
+        return Promise.resolve()
             .then(bobUploadsDeviceKeys)
-            .then(aliDownloadsKeys)
-            .nodeify(done);
+            .then(aliDownloadsKeys);
     });
 
-    it("Ali gets keys with an invalid signature", function(done) {
-        Promise.resolve()
+    it("Ali gets keys with an invalid signature", function() {
+        return Promise.resolve()
             .then(bobUploadsDeviceKeys)
             .then(function() {
                 // tamper bob's keys
@@ -455,11 +449,10 @@ describe("MatrixClient crypto", function() {
             }).then((devices) => {
                 // should get an empty list
                 expect(devices).toEqual([]);
-            })
-            .nodeify(done);
+            });
     });
 
-    it("Ali gets keys with an incorrect userId", function(done) {
+    it("Ali gets keys with an incorrect userId", function() {
         const eveUserId = "@eve:localhost";
 
         const bobDeviceKeys = {
@@ -488,7 +481,7 @@ describe("MatrixClient crypto", function() {
             return {device_keys: result};
         });
 
-        Promise.all([
+        return Promise.all([
             aliTestClient.client.downloadKeys([bobUserId, eveUserId]),
             aliTestClient.httpBackend.flush("/keys/query", 1),
         ]).then(function() {
@@ -496,14 +489,14 @@ describe("MatrixClient crypto", function() {
                 aliTestClient.client.getStoredDevicesForUser(bobUserId),
                 aliTestClient.client.getStoredDevicesForUser(eveUserId),
             ]);
-        }).spread((bobDevices, eveDevices) => {
+        }).then(([bobDevices, eveDevices]) => {
             // should get an empty list
             expect(bobDevices).toEqual([]);
             expect(eveDevices).toEqual([]);
-        }).nodeify(done);
+        });
     });
 
-    it("Ali gets keys with an incorrect deviceId", function(done) {
+    it("Ali gets keys with an incorrect deviceId", function() {
         const bobDeviceKeys = {
             algorithms: ['m.olm.v1.curve25519-aes-sha2', 'm.megolm.v1.aes-sha2'],
             device_id: 'bad_device',
@@ -530,7 +523,7 @@ describe("MatrixClient crypto", function() {
             return {device_keys: result};
         });
 
-        Promise.all([
+        return Promise.all([
             aliTestClient.client.downloadKeys([bobUserId]),
             aliTestClient.httpBackend.flush("/keys/query", 1),
         ]).then(function() {
@@ -538,7 +531,7 @@ describe("MatrixClient crypto", function() {
         }).then((devices) => {
             // should get an empty list
             expect(devices).toEqual([]);
-        }).nodeify(done);
+        });
     });
 
 
@@ -548,19 +541,18 @@ describe("MatrixClient crypto", function() {
             .then(() => bobTestClient.awaitOneTimeKeyUpload())
             .then((keys) => {
                 expect(Object.keys(keys).length).toEqual(5);
-                expect(Object.keys(bobTestClient.deviceKeys).length).toNotEqual(0);
+                expect(Object.keys(bobTestClient.deviceKeys).length).not.toEqual(0);
             });
     });
 
-    it("Ali sends a message", function(done) {
+    it("Ali sends a message", function() {
         aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
-        Promise.resolve()
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
             .then(aliEnablesEncryption)
-            .then(aliSendsFirstMessage)
-            .nodeify(done);
+            .then(aliSendsFirstMessage);
     });
 
     it("Bob receives a message", function() {
@@ -628,9 +620,9 @@ describe("MatrixClient crypto", function() {
             });
     });
 
-    it("Ali blocks Bob's device", function(done) {
+    it("Ali blocks Bob's device", function() {
         aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
-        Promise.resolve()
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -645,12 +637,12 @@ describe("MatrixClient crypto", function() {
                           expect(sentContent.ciphertext).toEqual({});
                       });
                 return Promise.all([p1, p2]);
-            }).nodeify(done);
+            });
     });
 
-    it("Bob receives two pre-key messages", function(done) {
+    it("Bob receives two pre-key messages", function() {
         aliTestClient.expectKeyQuery({device_keys: {[aliUserId]: {}}});
-        Promise.resolve()
+        return Promise.resolve()
             .then(() => aliTestClient.start())
             .then(() => bobTestClient.start())
             .then(() => firstSync(aliTestClient))
@@ -658,8 +650,7 @@ describe("MatrixClient crypto", function() {
             .then(aliSendsFirstMessage)
             .then(bobRecvMessage)
             .then(aliSendsMessage)
-            .then(bobRecvMessage)
-            .nodeify(done);
+            .then(bobRecvMessage);
     });
 
     it("Bob replies to the message", function() {
@@ -753,9 +744,9 @@ describe("MatrixClient crypto", function() {
             .then(() => httpBackend.when("POST", "/keys/upload")
                 .respond(200, (path, content) => {
                     expect(content.one_time_keys).toBeTruthy();
-                    expect(content.one_time_keys).toNotEqual({});
+                    expect(content.one_time_keys).not.toEqual({});
                     expect(Object.keys(content.one_time_keys).length)
-                        .toBeGreaterThanOrEqualTo(1);
+                        .toBeGreaterThanOrEqual(1);
                     logger.log('received %i one-time keys',
                                 Object.keys(content.one_time_keys).length);
                     // cancel futher calls by telling the client

spec/integ/matrix-client-event-emitter.spec.js

@@ -1,28 +1,16 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import * as utils from "../test-utils";
+import {TestClient} from "../TestClient";
 
 describe("MatrixClient events", function() {
-    const baseUrl = "http://localhost.or.something";
     let client;
     let httpBackend;
     const selfUserId = "@alice:localhost";
     const selfAccessToken = "aseukfgwef";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: selfUserId,
-            accessToken: selfAccessToken,
-        });
+        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+        client = testClient.client;
+        httpBackend = testClient.httpBackend;
         httpBackend.when("GET", "/pushrules").respond(200, {});
         httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
     });
@@ -164,7 +152,7 @@ describe("MatrixClient events", function() {
             });
             client.startClient();
 
-            httpBackend.flushAllExpected().done(function() {
+            httpBackend.flushAllExpected().then(function() {
                 expect(fired).toBe(true, "User.presence didn't fire.");
                 done();
             });
@@ -219,7 +207,7 @@ describe("MatrixClient events", function() {
             client.on("RoomState.events", function(event, state) {
                 eventsInvokeCount++;
                 const index = roomStateEventTypes.indexOf(event.getType());
-                expect(index).toNotEqual(
+                expect(index).not.toEqual(
                     -1, "Unexpected room state event type: " + event.getType(),
                 );
                 if (index >= 0) {

spec/integ/matrix-client-event-timeline.spec.js

@@ -1,13 +1,8 @@
-"use strict";
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-const EventTimeline = sdk.EventTimeline;
-import logger from '../../src/logger';
+import * as utils from "../test-utils";
+import {EventTimeline} from "../../src/matrix";
+import {logger} from "../../src/logger";
+import {TestClient} from "../TestClient";
 
-const baseUrl = "http://localhost.or.something";
 const userId = "@alice:localhost";
 const userName = "Alice";
 const accessToken = "aseukfgwef";
@@ -83,18 +78,19 @@ function startClient(httpBackend, client) {
     client.startClient();
 
     // set up a promise which will resolve once the client is initialised
-    const deferred = Promise.defer();
-    client.on("sync", function(state) {
-        logger.log("sync", state);
-        if (state != "SYNCING") {
-            return;
-        }
-        deferred.resolve();
+    const prom = new Promise((resolve) => {
+        client.on("sync", function(state) {
+            logger.log("sync", state);
+            if (state != "SYNCING") {
+                return;
+            }
+            resolve();
+        });
     });
 
     return Promise.all([
         httpBackend.flushAllExpected(),
-        deferred.promise,
+        prom,
     ]);
 }
 
@@ -103,9 +99,9 @@ describe("getEventTimeline support", function() {
     let client;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
+        const testClient = new TestClient(userId, "DEVICE", accessToken);
+        client = testClient.client;
+        httpBackend = testClient.httpBackend;
     });
 
     afterEach(function() {
@@ -115,53 +111,44 @@ describe("getEventTimeline support", function() {
         return httpBackend.stop();
     });
 
-    it("timeline support must be enabled to work", function(done) {
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-        });
-
-        startClient(httpBackend, client,
-        ).then(function() {
+    it("timeline support must be enabled to work", function() {
+        return startClient(httpBackend, client).then(function() {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             expect(function() {
                 client.getEventTimeline(timelineSet, "event");
             }).toThrow();
-        }).nodeify(done);
+        });
     });
 
     it("timeline support works when enabled", function() {
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-            timelineSupport: true,
-        });
+        const testClient = new TestClient(
+            userId,
+            "DEVICE",
+            accessToken,
+            undefined,
+            {timelineSupport: true},
+        );
+        client = testClient.client;
+        httpBackend = testClient.httpBackend;
 
         return startClient(httpBackend, client).then(() => {
             const room = client.getRoom(roomId);
             const timelineSet = room.getTimelineSets()[0];
             expect(function() {
                 client.getEventTimeline(timelineSet, "event");
-            }).toNotThrow();
+            }).not.toThrow();
         });
     });
 
 
     it("scrollback should be able to scroll back to before a gappy /sync",
-      function(done) {
+      function() {
         // need a client with timelineSupport disabled to make this work
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-        });
+
         let room;
 
-        startClient(httpBackend, client,
-        ).then(function() {
+        return startClient(httpBackend, client).then(function() {
             room = client.getRoom(roomId);
 
             httpBackend.when("GET", "/sync").respond(200, {
@@ -217,27 +204,24 @@ describe("getEventTimeline support", function() {
             expect(room.timeline[0].event).toEqual(EVENTS[0]);
             expect(room.timeline[1].event).toEqual(EVENTS[1]);
             expect(room.oldState.paginationToken).toEqual("pagin_end");
-        }).nodeify(done);
+        });
     });
 });
 
-import expect from 'expect';
-
 describe("MatrixClient event timelines", function() {
     let client = null;
     let httpBackend = null;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-            timelineSupport: true,
-        });
+        const testClient = new TestClient(
+            userId,
+            "DEVICE",
+            accessToken,
+            undefined,
+            {timelineSupport: true},
+        );
+        client = testClient.client;
+        httpBackend = testClient.httpBackend;
 
         return startClient(httpBackend, client);
     });
@@ -349,25 +333,25 @@ describe("MatrixClient event timelines", function() {
                     };
                 });
 
-            const deferred = Promise.defer();
-            client.on("sync", function() {
-                client.getEventTimeline(timelineSet, EVENTS[2].event_id,
-                ).then(function(tl) {
-                    expect(tl.getEvents().length).toEqual(4);
-                    expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
-                    expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
-                    expect(tl.getEvents()[3].event).toEqual(EVENTS[3]);
-                    expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
-                        .toEqual("start_token");
-                    // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
-                    //    .toEqual("s_5_4");
-                }).done(() => deferred.resolve(),
-                        (e) => deferred.reject(e));
+            const prom = new Promise((resolve, reject) => {
+                client.on("sync", function() {
+                    client.getEventTimeline(timelineSet, EVENTS[2].event_id,
+                    ).then(function(tl) {
+                        expect(tl.getEvents().length).toEqual(4);
+                        expect(tl.getEvents()[0].event).toEqual(EVENTS[1]);
+                        expect(tl.getEvents()[1].event).toEqual(EVENTS[2]);
+                        expect(tl.getEvents()[3].event).toEqual(EVENTS[3]);
+                        expect(tl.getPaginationToken(EventTimeline.BACKWARDS))
+                            .toEqual("start_token");
+                        // expect(tl.getPaginationToken(EventTimeline.FORWARDS))
+                        //    .toEqual("s_5_4");
+                    }).then(resolve, reject);
+                });
             });
 
             return Promise.all([
                 httpBackend.flushAllExpected(),
-                deferred.promise,
+                prom,
             ]);
         });
 
@@ -697,7 +681,7 @@ describe("MatrixClient event timelines", function() {
     });
 
 
-    it("should handle gappy syncs after redactions", function(done) {
+    it("should handle gappy syncs after redactions", function() {
         // https://github.com/vector-im/vector-web/issues/1389
 
         // a state event, followed by a redaction thereof
@@ -729,7 +713,7 @@ describe("MatrixClient event timelines", function() {
         };
         httpBackend.when("GET", "/sync").respond(200, syncData);
 
-        Promise.all([
+        return Promise.all([
             httpBackend.flushAllExpected(),
             utils.syncPromise(client),
         ]).then(function() {
@@ -765,6 +749,6 @@ describe("MatrixClient event timelines", function() {
             const room = client.getRoom(roomId);
             const tl = room.getLiveTimeline();
             expect(tl.getEvents().length).toEqual(1);
-        }).nodeify(done);
+        });
     });
 });

spec/integ/matrix-client-methods.spec.js

@@ -1,42 +1,23 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const publicGlobals = require("../../lib/matrix");
-const Room = publicGlobals.Room;
-const MemoryStore = publicGlobals.MemoryStore;
-const Filter = publicGlobals.Filter;
-const utils = require("../test-utils");
-const MockStorageApi = require("../MockStorageApi");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {CRYPTO_ENABLED} from "../../src/client";
+import {Filter, MemoryStore, Room} from "../../src/matrix";
+import {TestClient} from "../TestClient";
 
 describe("MatrixClient", function() {
-    const baseUrl = "http://localhost.or.something";
     let client = null;
     let httpBackend = null;
     let store = null;
-    let sessionStore = null;
     const userId = "@alice:localhost";
     const accessToken = "aseukfgwef";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        httpBackend = new HttpBackend();
         store = new MemoryStore();
 
-        const mockStorage = new MockStorageApi();
-        sessionStore = new sdk.WebStorageSessionStore(mockStorage);
-
-        sdk.request(httpBackend.requestFn);
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            deviceId: "aliceDevice",
-            accessToken: accessToken,
+        const testClient = new TestClient(userId, "aliceDevice", accessToken, undefined, {
             store: store,
-            sessionStore: sessionStore,
         });
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
     });
 
     afterEach(function() {
@@ -46,7 +27,7 @@ describe("MatrixClient", function() {
 
     describe("uploadContent", function() {
         const buf = new Buffer('hello world');
-        it("should upload the file", function(done) {
+        it("should upload the file", function() {
             httpBackend.when(
                 "POST", "/_matrix/media/r0/upload",
             ).check(function(req) {
@@ -74,25 +55,26 @@ describe("MatrixClient", function() {
             expect(uploads[0].promise).toBe(prom);
             expect(uploads[0].loaded).toEqual(0);
 
-            prom.then(function(response) {
+            const prom2 = prom.then(function(response) {
                 // for backwards compatibility, we return the raw JSON
                 expect(response).toEqual("content");
 
                 const uploads = client.getCurrentUploads();
                 expect(uploads.length).toEqual(0);
-            }).nodeify(done);
+            });
 
             httpBackend.flush();
+            return prom2;
         });
 
-        it("should parse the response if rawResponse=false", function(done) {
+        it("should parse the response if rawResponse=false", function() {
             httpBackend.when(
                 "POST", "/_matrix/media/r0/upload",
             ).check(function(req) {
                 expect(req.opts.json).toBeFalsy();
             }).respond(200, { "content_uri": "uri" });
 
-            client.uploadContent({
+            const prom = client.uploadContent({
                 stream: buf,
                 name: "hi.txt",
                 type: "text/plain",
@@ -100,12 +82,13 @@ describe("MatrixClient", function() {
                 rawResponse: false,
             }).then(function(response) {
                 expect(response.content_uri).toEqual("uri");
-            }).nodeify(done);
+            });
 
             httpBackend.flush();
+            return prom;
         });
 
-        it("should parse errors into a MatrixError", function(done) {
+        it("should parse errors into a MatrixError", function() {
             httpBackend.when(
                 "POST", "/_matrix/media/r0/upload",
             ).check(function(req) {
@@ -116,7 +99,7 @@ describe("MatrixClient", function() {
                 "error": "broken",
             });
 
-            client.uploadContent({
+            const prom = client.uploadContent({
                 stream: buf,
                 name: "hi.txt",
                 type: "text/plain",
@@ -126,12 +109,13 @@ describe("MatrixClient", function() {
                 expect(error.httpStatus).toEqual(400);
                 expect(error.errcode).toEqual("M_SNAFU");
                 expect(error.message).toEqual("broken");
-            }).nodeify(done);
+            });
 
             httpBackend.flush();
+            return prom;
         });
 
-        it("should return a promise which can be cancelled", function(done) {
+        it("should return a promise which can be cancelled", function() {
             const prom = client.uploadContent({
                 stream: buf,
                 name: "hi.txt",
@@ -143,17 +127,18 @@ describe("MatrixClient", function() {
             expect(uploads[0].promise).toBe(prom);
             expect(uploads[0].loaded).toEqual(0);
 
-            prom.then(function(response) {
+            const prom2 = prom.then(function(response) {
                 throw Error("request not aborted");
             }, function(error) {
                 expect(error).toEqual("aborted");
 
                 const uploads = client.getCurrentUploads();
                 expect(uploads.length).toEqual(0);
-            }).nodeify(done);
+            });
 
             const r = client.cancelUpload(prom);
             expect(r).toBe(true);
+            return prom2;
         });
     });
 
@@ -180,7 +165,7 @@ describe("MatrixClient", function() {
                 event_format: "client",
             });
             store.storeFilter(filter);
-            client.getFilter(userId, filterId, true).done(function(gotFilter) {
+            client.getFilter(userId, filterId, true).then(function(gotFilter) {
                 expect(gotFilter).toEqual(filter);
                 done();
             });
@@ -201,7 +186,7 @@ describe("MatrixClient", function() {
                 event_format: "client",
             });
             store.storeFilter(storeFilter);
-            client.getFilter(userId, filterId, false).done(function(gotFilter) {
+            client.getFilter(userId, filterId, false).then(function(gotFilter) {
                 expect(gotFilter.getDefinition()).toEqual(httpFilterDefinition);
                 done();
             });
@@ -219,7 +204,7 @@ describe("MatrixClient", function() {
             httpBackend.when(
                 "GET", "/user/" + encodeURIComponent(userId) + "/filter/" + filterId,
             ).respond(200, httpFilterDefinition);
-            client.getFilter(userId, filterId, true).done(function(gotFilter) {
+            client.getFilter(userId, filterId, true).then(function(gotFilter) {
                 expect(gotFilter.getDefinition()).toEqual(httpFilterDefinition);
                 expect(store.getFilter(userId, filterId)).toBeTruthy();
                 done();
@@ -247,7 +232,7 @@ describe("MatrixClient", function() {
                 filter_id: filterId,
             });
 
-            client.createFilter(filterDefinition).done(function(gotFilter) {
+            client.createFilter(filterDefinition).then(function(gotFilter) {
                 expect(gotFilter.getDefinition()).toEqual(filterDefinition);
                 expect(store.getFilter(userId, filterId)).toEqual(gotFilter);
                 done();
@@ -294,7 +279,7 @@ describe("MatrixClient", function() {
                 });
             }).respond(200, response);
 
-            httpBackend.flush().done(function() {
+            httpBackend.flush().then(function() {
                 done();
             });
         });
@@ -302,7 +287,7 @@ describe("MatrixClient", function() {
 
 
     describe("downloadKeys", function() {
-        if (!sdk.CRYPTO_ENABLED) {
+        if (!CRYPTO_ENABLED) {
             return;
         }
 
@@ -310,7 +295,7 @@ describe("MatrixClient", function() {
             return client.initCrypto();
         });
 
-        it("should do an HTTP request and then store the keys", function(done) {
+        it("should do an HTTP request and then store the keys", function() {
             const ed25519key = "7wG2lzAqbjcyEkOP7O4gU7ItYcn+chKzh5sT/5r2l78";
             // ed25519key = client.getDeviceEd25519Key();
             const borisKeys = {
@@ -362,8 +347,8 @@ describe("MatrixClient", function() {
 
             httpBackend.when("POST", "/keys/query").check(function(req) {
                 expect(req.data).toEqual({device_keys: {
-                    'boris': {},
-                    'chaz': {},
+                    'boris': [],
+                    'chaz': [],
                 }});
             }).respond(200, {
                 device_keys: {
@@ -372,7 +357,7 @@ describe("MatrixClient", function() {
                 },
             });
 
-            client.downloadKeys(["boris", "chaz"]).then(function(res) {
+            const prom = client.downloadKeys(["boris", "chaz"]).then(function(res) {
                 assertObjectContains(res.boris.dev1, {
                     verified: 0, // DeviceVerification.UNVERIFIED
                     keys: { "ed25519:dev1": ed25519key },
@@ -386,26 +371,26 @@ describe("MatrixClient", function() {
                     algorithms: ["2"],
                     unsigned: { "ghi": "def" },
                 });
-            }).nodeify(done);
+            });
 
             httpBackend.flush();
+            return prom;
         });
     });
 
     describe("deleteDevice", function() {
         const auth = {a: 1};
-        it("should pass through an auth dict", function(done) {
+        it("should pass through an auth dict", function() {
             httpBackend.when(
                 "DELETE", "/_matrix/client/r0/devices/my_device",
             ).check(function(req) {
                 expect(req.data).toEqual({auth: auth});
             }).respond(200);
 
-            client.deleteDevice(
-                "my_device", auth,
-            ).nodeify(done);
+            const prom = client.deleteDevice("my_device", auth);
 
             httpBackend.flush();
+            return prom;
         });
     });
 });

spec/integ/matrix-client-opts.spec.js

@@ -1,12 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const MatrixClient = sdk.MatrixClient;
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import * as utils from "../test-utils";
+import HttpBackend from "matrix-mock-request";
+import {MatrixClient} from "../../src/matrix";
+import {MatrixScheduler} from "../../src/scheduler";
+import {MemoryStore} from "../../src/store/memory";
+import {MatrixError} from "../../src/http-api";
 
 describe("MatrixClient opts", function() {
     const baseUrl = "http://localhost.or.something";
@@ -58,7 +55,6 @@ describe("MatrixClient opts", function() {
     };
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new HttpBackend();
     });
 
@@ -75,7 +71,7 @@ describe("MatrixClient opts", function() {
                 baseUrl: baseUrl,
                 userId: userId,
                 accessToken: accessToken,
-                scheduler: new sdk.MatrixScheduler(),
+                scheduler: new MatrixScheduler(),
             });
         });
 
@@ -88,7 +84,7 @@ describe("MatrixClient opts", function() {
             httpBackend.when("PUT", "/txn1").respond(200, {
                 event_id: eventId,
             });
-            client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
+            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(res.event_id).toEqual(eventId);
                 done();
             });
@@ -101,7 +97,7 @@ describe("MatrixClient opts", function() {
                 "m.room.create",
             ];
             client.on("event", function(event) {
-                expect(expectedEventTypes.indexOf(event.getType())).toNotEqual(
+                expect(expectedEventTypes.indexOf(event.getType())).not.toEqual(
                     -1, "Recv unexpected event type: " + event.getType(),
                 );
                 expectedEventTypes.splice(
@@ -128,7 +124,7 @@ describe("MatrixClient opts", function() {
         beforeEach(function() {
             client = new MatrixClient({
                 request: httpBackend.requestFn,
-                store: new sdk.MemoryStore(),
+                store: new MemoryStore(),
                 baseUrl: baseUrl,
                 userId: userId,
                 accessToken: accessToken,
@@ -137,11 +133,11 @@ describe("MatrixClient opts", function() {
         });
 
         it("shouldn't retry sending events", function(done) {
-            httpBackend.when("PUT", "/txn1").fail(500, {
+            httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                 errcode: "M_SOMETHING",
                 error: "Ruh roh",
-            });
-            client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
+            }));
+            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
             }, function(err) {
                 expect(err.errcode).toEqual("M_SOMETHING");
@@ -159,16 +155,16 @@ describe("MatrixClient opts", function() {
             });
             let sentA = false;
             let sentB = false;
-            client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
+            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 sentA = true;
                 expect(sentB).toBe(true);
             });
-            client.sendTextMessage("!foo:bar", "b body", "txn2").done(function(res) {
+            client.sendTextMessage("!foo:bar", "b body", "txn2").then(function(res) {
                 sentB = true;
                 expect(sentA).toBe(false);
             });
-            httpBackend.flush("/txn2", 1).done(function() {
-                httpBackend.flush("/txn1", 1).done(function() {
+            httpBackend.flush("/txn2", 1).then(function() {
+                httpBackend.flush("/txn1", 1).then(function() {
                     done();
                 });
             });
@@ -178,7 +174,7 @@ describe("MatrixClient opts", function() {
             httpBackend.when("PUT", "/txn1").respond(200, {
                 event_id: "foo",
             });
-            client.sendTextMessage("!foo:bar", "a body", "txn1").done(function(res) {
+            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                 expect(res.event_id).toEqual("foo");
                 done();
             });

spec/integ/matrix-client-retrying.spec.js

@@ -1,16 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-import Promise from 'bluebird';
-
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-const EventStatus = sdk.EventStatus;
-
-import expect from 'expect';
+import {EventStatus} from "../../src/matrix";
+import {MatrixScheduler} from "../../src/scheduler";
+import {Room} from "../../src/models/room";
+import {TestClient} from "../TestClient";
 
 describe("MatrixClient retrying", function() {
-    const baseUrl = "http://localhost.or.something";
     let client = null;
     let httpBackend = null;
     let scheduler;
@@ -20,17 +13,17 @@ describe("MatrixClient retrying", function() {
     let room;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-        scheduler = new sdk.MatrixScheduler();
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-            scheduler: scheduler,
-        });
-        room = new sdk.Room(roomId);
+        scheduler = new MatrixScheduler();
+        const testClient = new TestClient(
+            userId,
+            "DEVICE",
+            accessToken,
+            undefined,
+            {scheduler},
+        );
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
+        room = new Room(roomId);
         client.store.storeRoom(room);
     });
 
@@ -97,7 +90,7 @@ describe("MatrixClient retrying", function() {
         // wait for the localecho of ev1 to be updated
         const p3 = new Promise((resolve, reject) => {
             room.on("Room.localEchoUpdated", (ev0) => {
-                if(ev0 === ev1) {
+                if (ev0 === ev1) {
                     resolve();
                 }
             });

spec/integ/matrix-client-room-timeline.spec.js

@@ -1,15 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const EventStatus = sdk.EventStatus;
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
+import * as utils from "../test-utils";
+import {EventStatus} from "../../src/models/event";
+import {TestClient} from "../TestClient";
 
-import Promise from 'bluebird';
-import expect from 'expect';
 
 describe("MatrixClient room timelines", function() {
-    const baseUrl = "http://localhost.or.something";
     let client = null;
     let httpBackend = null;
     const userId = "@alice:localhost";
@@ -103,17 +97,18 @@ describe("MatrixClient room timelines", function() {
         });
     }
 
-    beforeEach(function(done) {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: userId,
-            accessToken: accessToken,
-            // these tests should work with or without timelineSupport
-            timelineSupport: true,
-        });
+    beforeEach(function() {
+        // these tests should work with or without timelineSupport
+        const testClient = new TestClient(
+            userId,
+            "DEVICE",
+            accessToken,
+            undefined,
+            {timelineSupport: true},
+        );
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
+
         setNextSyncData();
         httpBackend.when("GET", "/pushrules").respond(200, {});
         httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
@@ -122,9 +117,9 @@ describe("MatrixClient room timelines", function() {
             return NEXT_SYNC_DATA;
         });
         client.startClient();
-        httpBackend.flush("/pushrules").then(function() {
+        return httpBackend.flush("/pushrules").then(function() {
             return httpBackend.flush("/filter");
-        }).nodeify(done);
+        });
     });
 
     afterEach(function() {
@@ -153,7 +148,7 @@ describe("MatrixClient room timelines", function() {
                 expect(member.userId).toEqual(userId);
                 expect(member.name).toEqual(userName);
 
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(function() {
                     done();
                 });
             });
@@ -179,10 +174,10 @@ describe("MatrixClient room timelines", function() {
                     return;
                 }
                 const room = client.getRoom(roomId);
-                client.sendTextMessage(roomId, "I am a fish", "txn1").done(
+                client.sendTextMessage(roomId, "I am a fish", "txn1").then(
                 function() {
                     expect(room.timeline[1].getId()).toEqual(eventId);
-                    httpBackend.flush("/sync", 1).done(function() {
+                    httpBackend.flush("/sync", 1).then(function() {
                         expect(room.timeline[1].getId()).toEqual(eventId);
                         done();
                     });
@@ -212,10 +207,10 @@ describe("MatrixClient room timelines", function() {
                 }
                 const room = client.getRoom(roomId);
                 const promise = client.sendTextMessage(roomId, "I am a fish", "txn1");
-                httpBackend.flush("/sync", 1).done(function() {
+                httpBackend.flush("/sync", 1).then(function() {
                     expect(room.timeline.length).toEqual(2);
                     httpBackend.flush("/txn1", 1);
-                    promise.done(function() {
+                    promise.then(function() {
                         expect(room.timeline.length).toEqual(2);
                         expect(room.timeline[1].getId()).toEqual(eventId);
                         done();
@@ -250,7 +245,7 @@ describe("MatrixClient room timelines", function() {
                 const room = client.getRoom(roomId);
                 expect(room.timeline.length).toEqual(1);
 
-                client.scrollback(room).done(function() {
+                client.scrollback(room).then(function() {
                     expect(room.timeline.length).toEqual(1);
                     expect(room.oldState.paginationToken).toBe(null);
 
@@ -314,7 +309,7 @@ describe("MatrixClient room timelines", function() {
                 // sync response
                 expect(room.timeline.length).toEqual(1);
 
-                client.scrollback(room).done(function() {
+                client.scrollback(room).then(function() {
                     expect(room.timeline.length).toEqual(5);
                     const joinMsg = room.timeline[0];
                     expect(joinMsg.sender.name).toEqual("Old Alice");
@@ -352,7 +347,7 @@ describe("MatrixClient room timelines", function() {
                 const room = client.getRoom(roomId);
                 expect(room.timeline.length).toEqual(1);
 
-                client.scrollback(room).done(function() {
+                client.scrollback(room).then(function() {
                     expect(room.timeline.length).toEqual(3);
                     expect(room.timeline[0].event).toEqual(sbEvents[1]);
                     expect(room.timeline[1].event).toEqual(sbEvents[0]);
@@ -383,11 +378,11 @@ describe("MatrixClient room timelines", function() {
                 const room = client.getRoom(roomId);
                 expect(room.oldState.paginationToken).toBeTruthy();
 
-                client.scrollback(room, 1).done(function() {
+                client.scrollback(room, 1).then(function() {
                     expect(room.oldState.paginationToken).toEqual(sbEndTok);
                 });
 
-                httpBackend.flush("/messages", 1).done(function() {
+                httpBackend.flush("/messages", 1).then(function() {
                     // still have a sync to flush
                     httpBackend.flush("/sync", 1).then(() => {
                         done();

spec/integ/matrix-client-syncing.spec.js

@@ -1,16 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const HttpBackend = require("matrix-mock-request");
-const utils = require("../test-utils");
-const MatrixEvent = sdk.MatrixEvent;
-const EventTimeline = sdk.EventTimeline;
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import {MatrixEvent} from "../../src/models/event";
+import {EventTimeline} from "../../src/models/event-timeline";
+import * as utils from "../test-utils";
+import {TestClient} from "../TestClient";
 
 describe("MatrixClient syncing", function() {
-    const baseUrl = "http://localhost.or.something";
     let client = null;
     let httpBackend = null;
     const selfUserId = "@alice:localhost";
@@ -23,14 +16,9 @@ describe("MatrixClient syncing", function() {
     const roomTwo = "!bar:localhost";
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        httpBackend = new HttpBackend();
-        sdk.request(httpBackend.requestFn);
-        client = sdk.createClient({
-            baseUrl: baseUrl,
-            userId: selfUserId,
-            accessToken: selfAccessToken,
-        });
+        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+        httpBackend = testClient.httpBackend;
+        client = testClient.client;
         httpBackend.when("GET", "/pushrules").respond(200, {});
         httpBackend.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
     });
@@ -53,7 +41,7 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flushAllExpected().done(function() {
+            httpBackend.flushAllExpected().then(function() {
                 done();
             });
         });
@@ -67,7 +55,7 @@ describe("MatrixClient syncing", function() {
 
             client.startClient();
 
-            httpBackend.flushAllExpected().done(function() {
+            httpBackend.flushAllExpected().then(function() {
                 done();
             });
         });
@@ -528,7 +516,7 @@ describe("MatrixClient syncing", function() {
                 awaitSyncEvent(),
             ]).then(function() {
                 const room = client.getRoom(roomTwo);
-                expect(room).toExist();
+                expect(room).toBeDefined();
                 const tok = room.getLiveTimeline()
                     .getPaginationToken(EventTimeline.BACKWARDS);
                 expect(tok).toEqual("roomtwotok");
@@ -693,12 +681,12 @@ describe("MatrixClient syncing", function() {
                             include_leave: true }});
             }).respond(200, { filter_id: "another_id" });
 
-            const defer = Promise.defer();
-
-            httpBackend.when("GET", "/sync").check(function(req) {
-                expect(req.queryParams.filter).toEqual("another_id");
-                defer.resolve();
-            }).respond(200, {});
+            const prom = new Promise((resolve) => {
+                httpBackend.when("GET", "/sync").check(function(req) {
+                    expect(req.queryParams.filter).toEqual("another_id");
+                    resolve();
+                }).respond(200, {});
+            });
 
             client.syncLeftRooms();
 
@@ -709,7 +697,7 @@ describe("MatrixClient syncing", function() {
                     // flush the syncs
                     return httpBackend.flushAllExpected();
                 }),
-                defer.promise,
+                prom,
             ]);
         });
 

spec/integ/megolm-integ.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,16 +15,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-"use strict";
-
-const anotherjson = require('another-json');
-import Promise from 'bluebird';
-import expect from 'expect';
-
-const utils = require('../../lib/utils');
-const testUtils = require('../test-utils');
-const TestClient = require('../TestClient').default;
-import logger from '../../src/logger';
+import anotherjson from "another-json";
+import * as utils from "../../src/utils";
+import * as testUtils from "../test-utils";
+import {TestClient} from "../TestClient";
+import {logger} from "../../src/logger";
 
 const ROOM_ID = "!room:id";
 
@@ -283,8 +279,6 @@ describe("megolm", function() {
     }
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-
         aliceTestClient = new TestClient(
             "@alice:localhost", "xzcvb", "akjgkrgjs",
         );
@@ -352,7 +346,7 @@ describe("megolm", function() {
     });
 
     it("Alice receives a megolm message before the session keys", function() {
-        // https://github.com/vector-im/riot-web/issues/2273
+        // https://github.com/vector-im/element-web/issues/2273
         let roomKeyEncrypted;
 
         return aliceTestClient.start().then(() => {
@@ -621,6 +615,9 @@ describe("megolm", function() {
             ).respond(200, {
                 event_id: '$event_id',
             });
+            aliceTestClient.httpBackend.when(
+                'PUT', '/sendToDevice/org.matrix.room_key.withheld/',
+            ).respond(200, {});
 
             return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test'),
@@ -713,11 +710,14 @@ describe("megolm", function() {
                 'PUT', '/send/',
             ).respond(200, function(path, content) {
                 logger.log('/send:', content);
-                expect(content.session_id).toNotEqual(megolmSessionId);
+                expect(content.session_id).not.toEqual(megolmSessionId);
                 return {
                     event_id: '$event_id',
                 };
             });
+            aliceTestClient.httpBackend.when(
+                'PUT', '/sendToDevice/org.matrix.room_key.withheld/',
+            ).respond(200, {});
 
             return Promise.all([
                 aliceTestClient.client.sendTextMessage(ROOM_ID, 'test2'),
@@ -726,7 +726,7 @@ describe("megolm", function() {
         });
     });
 
-    // https://github.com/vector-im/riot-web/issues/2676
+    // https://github.com/vector-im/element-web/issues/2676
     it("Alice should send to her other devices", function() {
         // for this test, we make the testOlmAccount be another of Alice's devices.
         // it ought to get included in messages Alice sends.
@@ -971,4 +971,60 @@ describe("megolm", function() {
             expect(event.getContent().body).toEqual('42');
         });
     });
+
+    it("Alice receives an untrusted megolm key, only to receive the trusted one shortly after", function() {
+        const testClient = new TestClient(
+            "@alice:localhost", "device2", "access_token2",
+        );
+        const groupSession = new Olm.OutboundGroupSession();
+        groupSession.create();
+        const inboundGroupSession = new Olm.InboundGroupSession();
+        inboundGroupSession.create(groupSession.session_key());
+        const rawEvent = encryptMegolmEvent({
+            senderKey: testSenderKey,
+            groupSession: groupSession,
+            room_id: ROOM_ID,
+        });
+        return testClient.client.initCrypto().then(() => {
+            const keys = [{
+                room_id: ROOM_ID,
+                algorithm: 'm.megolm.v1.aes-sha2',
+                session_id: groupSession.session_id(),
+                session_key: inboundGroupSession.export_session(0),
+                sender_key: testSenderKey,
+            }];
+            return testClient.client.importRoomKeys(keys, { untrusted: true });
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                event: true,
+                ...rawEvent,
+                room: ROOM_ID,
+            });
+            return event.attemptDecryption(testClient.client._crypto, true).then(() => {
+                expect(event.isKeySourceUntrusted()).toBeTruthy();
+            });
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                type: 'm.room_key',
+                content: {
+                    room_id: ROOM_ID,
+                    algorithm: 'm.megolm.v1.aes-sha2',
+                    session_id: groupSession.session_id(),
+                    session_key: groupSession.session_key(),
+                },
+                event: true,
+            });
+            event._senderCurve25519Key = testSenderKey;
+            return testClient.client._crypto._onRoomKeyEvent(event);
+        }).then(() => {
+            const event = testUtils.mkEvent({
+                event: true,
+                ...rawEvent,
+                room: ROOM_ID,
+            });
+            return event.attemptDecryption(testClient.client._crypto, true).then(() => {
+                expect(event.isKeySourceUntrusted()).toBeFalsy();
+            });
+        });
+    });
 });

spec/olm-loader.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector creations Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +15,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import logger from '../src/logger';
+import {logger} from '../src/logger';
+import * as utils from "../src/utils";
 
 // try to load the olm library.
 try {
@@ -23,3 +25,11 @@ try {
 } catch (e) {
     logger.warn("unable to run crypto tests: libolm not available");
 }
+
+// also try to set node crypto
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    logger.log('nodejs was compiled without crypto support: some tests will fail');
+}

spec/test-utils.js

@@ -1,13 +1,8 @@
-"use strict";
-import expect from 'expect';
-import Promise from 'bluebird';
-
 // load olm before the sdk if possible
 import './olm-loader';
 
-import logger from '../src/logger';
-import sdk from '..';
-const MatrixEvent = sdk.MatrixEvent;
+import {logger} from '../src/logger';
+import {MatrixEvent} from "../src/models/event";
 
 /**
  * Return a promise that is resolved when the client next emits a
@@ -16,7 +11,7 @@ const MatrixEvent = sdk.MatrixEvent;
  * @param {Number=} count Number of syncs to wait for (default 1)
  * @return {Promise} Resolves once the client has emitted a SYNCING event
  */
-module.exports.syncPromise = function(client, count) {
+export function syncPromise(client, count) {
     if (count === undefined) {
         count = 1;
     }
@@ -27,7 +22,7 @@ module.exports.syncPromise = function(client, count) {
     const p = new Promise((resolve, reject) => {
         const cb = (state) => {
             logger.log(`${Date.now()} syncPromise(${count}): ${state}`);
-            if (state == 'SYNCING') {
+            if (state === 'SYNCING') {
                 resolve();
             } else {
                 client.once('sync', cb);
@@ -37,21 +32,9 @@ module.exports.syncPromise = function(client, count) {
     });
 
     return p.then(() => {
-        return module.exports.syncPromise(client, count-1);
+        return syncPromise(client, count-1);
     });
-};
-
-/**
- * Perform common actions before each test case, e.g. printing the test case
- * name to stdout.
- * @param {Mocha.Context} context  The test context
- */
-module.exports.beforeEach = function(context) {
-    const desc = context.currentTest.fullTitle();
-
-    logger.log(desc);
-    logger.log(new Array(1 + desc.length).join("="));
-};
+}
 
 /**
  * Create a spy for an object and automatically spy its methods.
@@ -59,7 +42,7 @@ module.exports.beforeEach = function(context) {
  * @param {string} name The name of the class
  * @return {Object} An instantiated object with spied methods/properties.
  */
-module.exports.mock = function(constr, name) {
+export function mock(constr, name) {
     // Based on
     // http://eclipsesource.com/blogs/2014/03/27/mocks-in-jasmine-tests/
     const HelperConstr = new Function(); // jshint ignore:line
@@ -71,7 +54,7 @@ module.exports.mock = function(constr, name) {
     for (const key in constr.prototype) { // eslint-disable-line guard-for-in
         try {
             if (constr.prototype[key] instanceof Function) {
-                result[key] = expect.createSpy();
+                result[key] = jest.fn();
             }
         } catch (ex) {
             // Direct access to some non-function fields of DOM prototypes may
@@ -80,7 +63,7 @@ module.exports.mock = function(constr, name) {
         }
     }
     return result;
-};
+}
 
 /**
  * Create an Event.
@@ -93,7 +76,7 @@ module.exports.mock = function(constr, name) {
  * @param {boolean} opts.event True to make a MatrixEvent.
  * @return {Object} a JSON object representing this event.
  */
-module.exports.mkEvent = function(opts) {
+export function mkEvent(opts) {
     if (!opts.type || !opts.content) {
         throw new Error("Missing .type or .content =>" + JSON.stringify(opts));
     }
@@ -112,14 +95,14 @@ module.exports.mkEvent = function(opts) {
         event.state_key = "";
     }
     return opts.event ? new MatrixEvent(event) : event;
-};
+}
 
 /**
  * Create an m.presence event.
  * @param {Object} opts Values for the presence.
  * @return {Object|MatrixEvent} The event
  */
-module.exports.mkPresence = function(opts) {
+export function mkPresence(opts) {
     if (!opts.user) {
         throw new Error("Missing user");
     }
@@ -135,7 +118,7 @@ module.exports.mkPresence = function(opts) {
         },
     };
     return opts.event ? new MatrixEvent(event) : event;
-};
+}
 
 /**
  * Create an m.room.member event.
@@ -150,7 +133,7 @@ module.exports.mkPresence = function(opts) {
  * @param {boolean} opts.event True to make a MatrixEvent.
  * @return {Object|MatrixEvent} The event
  */
-module.exports.mkMembership = function(opts) {
+export function mkMembership(opts) {
     opts.type = "m.room.member";
     if (!opts.skey) {
         opts.skey = opts.sender || opts.user;
@@ -167,8 +150,8 @@ module.exports.mkMembership = function(opts) {
     if (opts.url) {
         opts.content.avatar_url = opts.url;
     }
-    return module.exports.mkEvent(opts);
-};
+    return mkEvent(opts);
+}
 
 /**
  * Create an m.room.message event.
@@ -179,7 +162,7 @@ module.exports.mkMembership = function(opts) {
  * @param {boolean} opts.event True to make a MatrixEvent.
  * @return {Object|MatrixEvent} The event
  */
-module.exports.mkMessage = function(opts) {
+export function mkMessage(opts) {
     opts.type = "m.room.message";
     if (!opts.msg) {
         opts.msg = "Random->" + Math.random();
@@ -191,8 +174,8 @@ module.exports.mkMessage = function(opts) {
         msgtype: "m.text",
         body: opts.msg,
     };
-    return module.exports.mkEvent(opts);
-};
+    return mkEvent(opts);
+}
 
 
 /**
@@ -200,10 +183,10 @@ module.exports.mkMessage = function(opts) {
  *
  * @constructor
  */
-module.exports.MockStorageApi = function() {
+export function MockStorageApi() {
     this.data = {};
-};
-module.exports.MockStorageApi.prototype = {
+}
+MockStorageApi.prototype = {
     get length() {
         return Object.keys(this.data).length;
     },
@@ -228,7 +211,7 @@ module.exports.MockStorageApi.prototype = {
  * @param {MatrixEvent} event
  * @returns {Promise} promise which resolves (to `event`) when the event has been decrypted
  */
-module.exports.awaitDecryption = function(event) {
+export function awaitDecryption(event) {
     if (!event.isBeingDecrypted()) {
         return Promise.resolve(event);
     }
@@ -241,4 +224,145 @@ module.exports.awaitDecryption = function(event) {
             resolve(ev);
         });
     });
+}
+
+
+export function HttpResponse(
+    httpLookups, acceptKeepalives, ignoreUnhandledSync,
+) {
+    this.httpLookups = httpLookups;
+    this.acceptKeepalives = acceptKeepalives === undefined ? true : acceptKeepalives;
+    this.ignoreUnhandledSync = ignoreUnhandledSync;
+    this.pendingLookup = null;
+}
+
+HttpResponse.prototype.request = function(
+    cb, method, path, qp, data, prefix,
+) {
+    if (path === HttpResponse.KEEP_ALIVE_PATH && this.acceptKeepalives) {
+        return Promise.resolve();
+    }
+    const next = this.httpLookups.shift();
+    const logLine = (
+        "MatrixClient[UT] RECV " + method + " " + path + "  " +
+            "EXPECT " + (next ? next.method : next) + " " + (next ? next.path : next)
+    );
+    logger.log(logLine);
+
+    if (!next) { // no more things to return
+        if (method === "GET" && path === "/sync" && this.ignoreUnhandledSync) {
+            logger.log("MatrixClient[UT] Ignoring.");
+            return new Promise(() => {});
+        }
+        if (this.pendingLookup) {
+            if (this.pendingLookup.method === method
+                && this.pendingLookup.path === path) {
+                return this.pendingLookup.promise;
+            }
+            // >1 pending thing, and they are different, whine.
+            expect(false).toBe(
+                true, ">1 pending request. You should probably handle them. " +
+                    "PENDING: " + JSON.stringify(this.pendingLookup) + " JUST GOT: " +
+                    method + " " + path,
+            );
+        }
+        this.pendingLookup = {
+            promise: new Promise(() => {}),
+            method: method,
+            path: path,
+        };
+        return this.pendingLookup.promise;
+    }
+    if (next.path === path && next.method === method) {
+        logger.log(
+            "MatrixClient[UT] Matched. Returning " +
+                (next.error ? "BAD" : "GOOD") + " response",
+        );
+        if (next.expectBody) {
+            expect(next.expectBody).toEqual(data);
+        }
+        if (next.expectQueryParams) {
+            Object.keys(next.expectQueryParams).forEach(function(k) {
+                expect(qp[k]).toEqual(next.expectQueryParams[k]);
+            });
+        }
+
+        if (next.thenCall) {
+            process.nextTick(next.thenCall, 0); // next tick so we return first.
+        }
+
+        if (next.error) {
+            return Promise.reject({
+                errcode: next.error.errcode,
+                httpStatus: next.error.httpStatus,
+                name: next.error.errcode,
+                message: "Expected testing error",
+                data: next.error,
+            });
+        }
+        return Promise.resolve(next.data);
+    } else if (method === "GET" && path === "/sync" && this.ignoreUnhandledSync) {
+        logger.log("MatrixClient[UT] Ignoring.");
+        this.httpLookups.unshift(next);
+        return new Promise(() => {});
+    }
+    expect(true).toBe(false, "Expected different request. " + logLine);
+    return new Promise(() => {});
 };
+
+HttpResponse.KEEP_ALIVE_PATH = "/_matrix/client/versions";
+
+HttpResponse.PUSH_RULES_RESPONSE = {
+    method: "GET",
+    path: "/pushrules/",
+    data: {},
+};
+
+HttpResponse.USER_ID = "@alice:bar";
+
+HttpResponse.filterResponse = function(userId) {
+    const filterPath = "/user/" + encodeURIComponent(userId) + "/filter";
+    return {
+        method: "POST",
+        path: filterPath,
+        data: { filter_id: "f1lt3r" },
+    };
+};
+
+HttpResponse.SYNC_DATA = {
+    next_batch: "s_5_3",
+    presence: { events: [] },
+    rooms: {},
+};
+
+HttpResponse.SYNC_RESPONSE = {
+    method: "GET",
+    path: "/sync",
+    data: HttpResponse.SYNC_DATA,
+};
+
+HttpResponse.defaultResponses = function(userId) {
+    return [
+        HttpResponse.PUSH_RULES_RESPONSE,
+        HttpResponse.filterResponse(userId),
+        HttpResponse.SYNC_RESPONSE,
+    ];
+};
+
+export function setHttpResponses(
+    client, responses, acceptKeepalives, ignoreUnhandledSyncs,
+) {
+    const httpResponseObj = new HttpResponse(
+        responses, acceptKeepalives, ignoreUnhandledSyncs,
+    );
+
+    const httpReq = httpResponseObj.request.bind(httpResponseObj);
+    client._http = [
+        "authedRequest", "authedRequestWithPrefix", "getContentUri",
+        "request", "requestWithPrefix", "uploadContent",
+    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
+    client._http.authedRequest.mockImplementation(httpReq);
+    client._http.authedRequestWithPrefix.mockImplementation(httpReq);
+    client._http.requestWithPrefix.mockImplementation(httpReq);
+    client._http.request.mockImplementation(httpReq);
+}

spec/unit/ReEmitter.spec.ts

@@ -0,0 +1,72 @@
+/*
+Copyright 2021 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+import { ReEmitter } from "../../src/ReEmitter";
+
+const EVENTNAME = "UnknownEntry";
+
+class EventSource extends EventEmitter {
+    doTheThing() {
+        this.emit(EVENTNAME, "foo", "bar");
+    }
+
+    doAnError() {
+        this.emit('error');
+    }
+}
+
+class EventTarget extends EventEmitter {
+
+}
+
+describe("ReEmitter", function() {
+    it("Re-Emits events with the same args", function() {
+        const src = new EventSource();
+        const tgt = new EventTarget();
+
+        const handler = jest.fn();
+        tgt.on(EVENTNAME, handler);
+
+        const reEmitter = new ReEmitter(tgt);
+        reEmitter.reEmit(src, [EVENTNAME]);
+
+        src.doTheThing();
+
+        // Args should be the args passed to 'emit' after the event name, and
+        // also the source object of the event which re-emitter adds
+        expect(handler).toHaveBeenCalledWith("foo", "bar", src);
+    });
+
+    it("Doesn't throw if no handler for 'error' event", function() {
+        const src = new EventSource();
+        const tgt = new EventTarget();
+
+        const reEmitter = new ReEmitter(tgt);
+        reEmitter.reEmit(src, ['error']);
+
+        // without the workaround in ReEmitter, this would throw
+        src.doAnError();
+
+        const handler = jest.fn();
+        tgt.on('error', handler);
+
+        src.doAnError();
+
+        // Now we've attached an error handler, it should be called
+        expect(handler).toHaveBeenCalled();
+    });
+});

spec/unit/autodiscovery.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,24 +14,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const utils = require("../test-utils");
-
-const AutoDiscovery = sdk.AutoDiscovery;
-
-import expect from 'expect';
 import MockHttpBackend from "matrix-mock-request";
-
+import * as sdk from "../../src";
+import {AutoDiscovery} from "../../src/autodiscovery";
 
 describe("AutoDiscovery", function() {
     let httpBackend = null;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         httpBackend = new MockHttpBackend();
         sdk.request(httpBackend.requestFn);
     });
@@ -416,8 +408,8 @@ describe("AutoDiscovery", function() {
         ]);
     });
 
-    it("should return FAIL_ERROR when the identity server configuration is wrong " +
-        "(missing base_url)", function() {
+    it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
+        "is wrong (missing base_url)", function() {
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
             expect(req.opts.uri)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
@@ -438,14 +430,14 @@ describe("AutoDiscovery", function() {
             AutoDiscovery.findClientConfig("example.org").then((conf) => {
                 const expected = {
                     "m.homeserver": {
-                        state: "FAIL_ERROR",
-                        error: AutoDiscovery.ERROR_INVALID_IS,
+                        state: "SUCCESS",
+                        error: null,
 
                         // We still expect the base_url to be here for debugging purposes.
                         base_url: "https://chat.example.org",
                     },
                     "m.identity_server": {
-                        state: "FAIL_ERROR",
+                        state: "FAIL_PROMPT",
                         error: AutoDiscovery.ERROR_INVALID_IS_BASE_URL,
                         base_url: null,
                     },
@@ -456,8 +448,8 @@ describe("AutoDiscovery", function() {
         ]);
     });
 
-    it("should return FAIL_ERROR when the identity server configuration is wrong " +
-        "(empty base_url)", function() {
+    it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
+        "is wrong (empty base_url)", function() {
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
             expect(req.opts.uri)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
@@ -478,14 +470,14 @@ describe("AutoDiscovery", function() {
             AutoDiscovery.findClientConfig("example.org").then((conf) => {
                 const expected = {
                     "m.homeserver": {
-                        state: "FAIL_ERROR",
-                        error: AutoDiscovery.ERROR_INVALID_IS,
+                        state: "SUCCESS",
+                        error: null,
 
                         // We still expect the base_url to be here for debugging purposes.
                         base_url: "https://chat.example.org",
                     },
                     "m.identity_server": {
-                        state: "FAIL_ERROR",
+                        state: "FAIL_PROMPT",
                         error: AutoDiscovery.ERROR_INVALID_IS_BASE_URL,
                         base_url: null,
                     },
@@ -496,8 +488,8 @@ describe("AutoDiscovery", function() {
         ]);
     });
 
-    it("should return FAIL_ERROR when the identity server configuration is wrong " +
-        "(validation error: 404)", function() {
+    it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
+        "is wrong (validation error: 404)", function() {
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
             expect(req.opts.uri)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
@@ -519,14 +511,14 @@ describe("AutoDiscovery", function() {
             AutoDiscovery.findClientConfig("example.org").then((conf) => {
                 const expected = {
                     "m.homeserver": {
-                        state: "FAIL_ERROR",
-                        error: AutoDiscovery.ERROR_INVALID_IS,
+                        state: "SUCCESS",
+                        error: null,
 
                         // We still expect the base_url to be here for debugging purposes.
                         base_url: "https://chat.example.org",
                     },
                     "m.identity_server": {
-                        state: "FAIL_ERROR",
+                        state: "FAIL_PROMPT",
                         error: AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER,
                         base_url: "https://identity.example.org",
                     },
@@ -537,8 +529,8 @@ describe("AutoDiscovery", function() {
         ]);
     });
 
-    it("should return FAIL_ERROR when the identity server configuration is wrong " +
-        "(validation error: 500)", function() {
+    it("should return SUCCESS / FAIL_PROMPT when the identity server configuration " +
+        "is wrong (validation error: 500)", function() {
         httpBackend.when("GET", "/_matrix/client/versions").check((req) => {
             expect(req.opts.uri)
                 .toEqual("https://chat.example.org/_matrix/client/versions");
@@ -560,14 +552,14 @@ describe("AutoDiscovery", function() {
             AutoDiscovery.findClientConfig("example.org").then((conf) => {
                 const expected = {
                     "m.homeserver": {
-                        state: "FAIL_ERROR",
-                        error: AutoDiscovery.ERROR_INVALID_IS,
+                        state: "SUCCESS",
+                        error: null,
 
                         // We still expect the base_url to be here for debugging purposes
                         base_url: "https://chat.example.org",
                     },
                     "m.identity_server": {
-                        state: "FAIL_ERROR",
+                        state: "FAIL_PROMPT",
                         error: AutoDiscovery.ERROR_INVALID_IDENTITY_SERVER,
                         base_url: "https://identity.example.org",
                     },

spec/unit/content-repo.spec.js

@@ -1,22 +1,13 @@
-"use strict";
-import 'source-map-support/register';
-const ContentRepo = require("../../lib/content-repo");
-const testUtils = require("../test-utils");
-
-import expect from 'expect';
+import {getHttpUriForMxc} from "../../src/content-repo";
 
 describe("ContentRepo", function() {
     const baseUrl = "https://my.home.server";
 
-    beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-    });
-
     describe("getHttpUriForMxc", function() {
         it("should do nothing to HTTP URLs when allowing direct links", function() {
             const httpUrl = "http://example.com/image.jpeg";
             expect(
-                ContentRepo.getHttpUriForMxc(
+                getHttpUriForMxc(
                     baseUrl, httpUrl, undefined, undefined, undefined, true,
                 ),
             ).toEqual(httpUrl);
@@ -24,25 +15,25 @@ describe("ContentRepo", function() {
 
         it("should return the empty string HTTP URLs by default", function() {
             const httpUrl = "http://example.com/image.jpeg";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, httpUrl)).toEqual("");
+            expect(getHttpUriForMxc(baseUrl, httpUrl)).toEqual("");
         });
 
         it("should return a download URL if no width/height/resize are specified",
         function() {
             const mxcUri = "mxc://server.name/resourceid";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
+            expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
                 baseUrl + "/_matrix/media/r0/download/server.name/resourceid",
             );
         });
 
         it("should return the empty string for null input", function() {
-            expect(ContentRepo.getHttpUriForMxc(null)).toEqual("");
+            expect(getHttpUriForMxc(null)).toEqual("");
         });
 
         it("should return a thumbnail URL if a width/height/resize is specified",
         function() {
             const mxcUri = "mxc://server.name/resourceid";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri, 32, 64, "crop")).toEqual(
+            expect(getHttpUriForMxc(baseUrl, mxcUri, 32, 64, "crop")).toEqual(
                 baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" +
                 "?width=32&height=64&method=crop",
             );
@@ -51,7 +42,7 @@ describe("ContentRepo", function() {
         it("should put fragments from mxc:// URIs after any query parameters",
         function() {
             const mxcUri = "mxc://server.name/resourceid#automade";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri, 32)).toEqual(
+            expect(getHttpUriForMxc(baseUrl, mxcUri, 32)).toEqual(
                 baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" +
                 "?width=32#automade",
             );
@@ -60,36 +51,9 @@ describe("ContentRepo", function() {
         it("should put fragments from mxc:// URIs at the end of the HTTP URI",
         function() {
             const mxcUri = "mxc://server.name/resourceid#automade";
-            expect(ContentRepo.getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
+            expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
                 baseUrl + "/_matrix/media/r0/download/server.name/resourceid#automade",
             );
         });
     });
-
-    describe("getIdenticonUri", function() {
-        it("should do nothing for null input", function() {
-            expect(ContentRepo.getIdenticonUri(null)).toEqual(null);
-        });
-
-        it("should set w/h by default to 96", function() {
-            expect(ContentRepo.getIdenticonUri(baseUrl, "foobar")).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=96&height=96",
-            );
-        });
-
-        it("should be able to set custom w/h", function() {
-            expect(ContentRepo.getIdenticonUri(baseUrl, "foobar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=32&height=64",
-            );
-        });
-
-        it("should URL encode the identicon string", function() {
-            expect(ContentRepo.getIdenticonUri(baseUrl, "foo#bar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foo%23bar" +
-                "?width=32&height=64",
-            );
-        });
-    });
 });

spec/unit/crypto.spec.js

@@ -1,38 +1,92 @@
-import 'source-map-support/register';
-
 import '../olm-loader';
-
-import Crypto from '../../lib/crypto';
-import expect from 'expect';
-
-import WebStorageSessionStore from '../../lib/store/session/webstorage';
-import MemoryCryptoStore from '../../lib/crypto/store/memory-crypto-store.js';
-import MockStorageApi from '../MockStorageApi';
-import TestClient from '../TestClient';
-import {MatrixEvent} from '../../lib/models/event';
-import Room from '../../lib/models/room';
-import olmlib from '../../lib/crypto/olmlib';
-import lolex from 'lolex';
-
-const EventEmitter = require("events").EventEmitter;
-
-const sdk = require("../..");
+import {Crypto} from "../../src/crypto";
+import {WebStorageSessionStore} from "../../src/store/session/webstorage";
+import {MemoryCryptoStore} from "../../src/crypto/store/memory-crypto-store";
+import {MockStorageApi} from "../MockStorageApi";
+import {TestClient} from "../TestClient";
+import {MatrixEvent} from "../../src/models/event";
+import {Room} from "../../src/models/room";
+import * as olmlib from "../../src/crypto/olmlib";
+import {sleep} from "../../src/utils";
+import {EventEmitter} from "events";
+import {CRYPTO_ENABLED} from "../../src/client";
+import {DeviceInfo} from "../../src/crypto/deviceinfo";
 
 const Olm = global.Olm;
 
 describe("Crypto", function() {
-    if (!sdk.CRYPTO_ENABLED) {
+    if (!CRYPTO_ENABLED) {
         return;
     }
 
-    beforeEach(function(done) {
-        Olm.init().then(done);
+    beforeAll(function() {
+        return Olm.init();
     });
 
     it("Crypto exposes the correct olm library version", function() {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 
+    describe("encrypted events", function() {
+        it("provides encryption information", async function() {
+            const client = (new TestClient(
+                "@alice:example.com", "deviceid",
+            )).client;
+            await client.initCrypto();
+
+            // unencrypted event
+            const event = {
+                getId: () => "$event_id",
+                getSenderKey: () => null,
+                getWireContent: () => {return {};},
+            };
+
+            let encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeFalsy();
+
+            // unknown sender (e.g. deleted device), forwarded megolm key (untrusted)
+            event.getSenderKey = () => 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            event.getWireContent = () => {return {algorithm: olmlib.MEGOLM_ALGORITHM};};
+            event.getForwardingCurve25519KeyChain = () => ["not empty"];
+            event.isKeySourceUntrusted = () => false;
+            event.getClaimedEd25519Key =
+                () => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeFalsy();
+
+            // known sender, megolm key from backup
+            event.getForwardingCurve25519KeyChain = () => [];
+            event.isKeySourceUntrusted = () => true;
+            const device = new DeviceInfo("FLIBBLE");
+            device.keys["curve25519:FLIBBLE"] =
+                'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            device.keys["ed25519:FLIBBLE"] =
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+            client._crypto._deviceList.getDeviceByIdentityKey = () => device;
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeFalsy();
+
+            // known sender, trusted megolm key, but bad ed25519key
+            event.isKeySourceUntrusted = () => false;
+            device.keys["ed25519:FLIBBLE"] =
+                'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeTruthy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeTruthy();
+
+            client.stopClient();
+        });
+    });
 
     describe('Session management', function() {
         const otkResponse = {
@@ -76,9 +130,9 @@ describe("Crypto", function() {
             });
 
             mockBaseApis = {
-                sendToDevice: expect.createSpy(),
-                getKeyBackupVersion: expect.createSpy(),
-                isGuest: expect.createSpy(),
+                sendToDevice: jest.fn(),
+                getKeyBackupVersion: jest.fn(),
+                isGuest: jest.fn(),
             };
             mockRoomList = {};
 
@@ -110,15 +164,16 @@ describe("Crypto", function() {
             });
 
             fakeEmitter.emit('toDeviceEvent', {
-                getType: expect.createSpy().andReturn('m.room.message'),
-                getContent: expect.createSpy().andReturn({
+                getId: jest.fn().mockReturnValue("$wedged"),
+                getType: jest.fn().mockReturnValue('m.room.message'),
+                getContent: jest.fn().mockReturnValue({
                     msgtype: 'm.bad.encrypted',
                 }),
-                getWireContent: expect.createSpy().andReturn({
+                getWireContent: jest.fn().mockReturnValue({
                     algorithm: 'm.olm.v1.curve25519-aes-sha2',
                     sender_key: 'this is a key',
                 }),
-                getSender: expect.createSpy().andReturn('@bob:home.server'),
+                getSender: jest.fn().mockReturnValue('@bob:home.server'),
             });
 
             await prom;
@@ -245,7 +300,7 @@ describe("Crypto", function() {
                 await bobDecryptor.onRoomKeyEvent(ksEvent);
                 await eventPromise;
                 expect(events[0].getContent().msgtype).toBe("m.bad.encrypted");
-                expect(events[1].getContent().msgtype).toNotBe("m.bad.encrypted");
+                expect(events[1].getContent().msgtype).not.toBe("m.bad.encrypted");
 
                 const cryptoStore = bobClient._cryptoStore;
                 const eventContent = events[0].getWireContent();
@@ -260,7 +315,7 @@ describe("Crypto", function() {
                 // the room key request should still be there, since we haven't
                 // decrypted everything
                 expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
-                    .toExist();
+                    .toBeDefined();
 
                 // keyshare the session key starting at the first message, so
                 // that it can now be decrypted
@@ -268,10 +323,11 @@ describe("Crypto", function() {
                 ksEvent = await keyshareEventForEvent(events[0], 0);
                 await bobDecryptor.onRoomKeyEvent(ksEvent);
                 await eventPromise;
-                expect(events[0].getContent().msgtype).toNotBe("m.bad.encrypted");
-                // the room key request should be gone since we've now decypted everything
+                expect(events[0].getContent().msgtype).not.toBe("m.bad.encrypted");
+                await sleep(1);
+                // the room key request should be gone since we've now decrypted everything
                 expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
-                    .toNotExist();
+                    .toBeFalsy();
             },
         );
 
@@ -296,10 +352,12 @@ describe("Crypto", function() {
                 sender_key: "senderkey",
             };
             expect(await cryptoStore.getOutgoingRoomKeyRequest(roomKeyRequestBody))
-                .toExist();
+                .toBeDefined();
         });
 
         it("uses a new txnid for re-requesting keys", async function() {
+            jest.useFakeTimers();
+
             const event = new MatrixEvent({
                 sender: "@bob:example.com",
                 room_id: "!someroom",
@@ -309,58 +367,35 @@ describe("Crypto", function() {
                     sender_key: "senderkey",
                 },
             });
-            /* return a promise and a function. When the function is called,
-             * the promise will be resolved.
-             */
-            function awaitFunctionCall() {
-                let func;
-                const promise = new Promise((resolve, reject) => {
-                    func = function(...args) {
-                        resolve(args);
-                        return new Promise((resolve, reject) => {
-                            // give us some time to process the result before
-                            // continuing
-                            global.setTimeout(resolve, 1);
-                        });
-                    };
-                });
-                return {func, promise};
-            }
-
+            // replace Alice's sendToDevice function with a mock
+            aliceClient.sendToDevice = jest.fn().mockResolvedValue(undefined);
             aliceClient.startClient();
 
-            const clock = lolex.install();
+            // make a room key request, and record the transaction ID for the
+            // sendToDevice call
+            await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            // key requests get queued until the sync has finished, but we don't
+            // let the client set up enough for that to happen, so gut-wrench a bit
+            // to force it to send now.
+            aliceClient._crypto._outgoingRoomKeyRequestManager.sendQueuedRequests();
+            jest.runAllTimers();
+            await Promise.resolve();
+            expect(aliceClient.sendToDevice).toBeCalledTimes(1);
+            const txnId = aliceClient.sendToDevice.mock.calls[0][2];
 
-            try {
-                let promise;
-                // make a room key request, and record the transaction ID for the
-                // sendToDevice call
-                ({promise, func: aliceClient.sendToDevice} = awaitFunctionCall());
-                await aliceClient.cancelAndResendEventRoomKeyRequest(event);
-                clock.runToLast();
-                let args = await promise;
-                const txnId = args[2];
-                clock.runToLast();
+            // give the room key request manager time to update the state
+            // of the request
+            await Promise.resolve();
 
-                // give the room key request manager time to update the state
-                // of the request
-                await Promise.resolve();
-
-                // cancel and resend the room key request
-                ({promise, func: aliceClient.sendToDevice} = awaitFunctionCall());
-                await aliceClient.cancelAndResendEventRoomKeyRequest(event);
-                clock.runToLast();
-                // the first call to sendToDevice will be the cancellation
-                args = await promise;
-                // the second call to sendToDevice will be the key request
-                ({promise, func: aliceClient.sendToDevice} = awaitFunctionCall());
-                clock.runToLast();
-                args = await promise;
-                clock.runToLast();
-                expect(args[2]).toNotBe(txnId);
-            } finally {
-                clock.uninstall();
-            }
+            // cancel and resend the room key request
+            await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            jest.runAllTimers();
+            await Promise.resolve();
+            // cancelAndResend will call sendToDevice twice:
+            // the first call to sendToDevice will be the cancellation
+            // the second call to sendToDevice will be the key request
+            expect(aliceClient.sendToDevice).toBeCalledTimes(3);
+            expect(aliceClient.sendToDevice.mock.calls[2][2]).not.toBe(txnId);
         });
     });
 });

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -0,0 +1,258 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+import {
+    CrossSigningInfo,
+    createCryptoStoreCacheCallbacks,
+} from '../../../src/crypto/CrossSigning';
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+import {OlmDevice} from "../../../src/crypto/OlmDevice";
+import {logger} from '../../../src/logger';
+
+const userId = "@alice:example.com";
+
+// Private key for tests only
+const testKey = new Uint8Array([
+    0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+    0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+    0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+    0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+]);
+
+const types = [
+    { type: "master", shouldCache: true },
+    { type: "self_signing", shouldCache: true },
+    { type: "user_signing", shouldCache: true },
+    { type: "invalid", shouldCache: false },
+];
+
+const badKey = Uint8Array.from(testKey);
+badKey[0] ^= 1;
+
+const masterKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
+
+describe("CrossSigningInfo.getCrossSigningKey", function() {
+    if (!global.Olm) {
+        logger.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
+    it("should throw if no callback is provided", async () => {
+        const info = new CrossSigningInfo(userId);
+        await expect(info.getCrossSigningKey("master")).rejects.toThrow();
+    });
+
+    it.each(types)("should throw if the callback returns falsey",
+                   async ({type, shouldCache}) => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => false,
+        });
+        await expect(info.getCrossSigningKey(type)).rejects.toThrow("falsey");
+    });
+
+    it("should throw if the expected key doesn't come back", async () => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => masterKeyPub,
+        });
+        await expect(info.getCrossSigningKey("master", "")).rejects.toThrow();
+    });
+
+    it("should return a key from its callback", async () => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => testKey,
+        });
+        const [pubKey, pkSigning] = await info.getCrossSigningKey("master", masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        // check that the pkSigning object corresponds to the pubKey
+        const signature = pkSigning.sign("message");
+        const util = new global.Olm.Utility();
+        try {
+            util.ed25519_verify(pubKey, "message", signature);
+        } finally {
+            util.free();
+        }
+    });
+
+    it.each(types)("should request a key from the cache callback (if set)" +
+                   " and does not call app if one is found" +
+                   " %o",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockImplementation(() => {
+            if (shouldCache) {
+                return Promise.reject(new Error("Regular callback called"));
+            } else {
+                return Promise.resolve(testKey);
+            }
+        });
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+        if (shouldCache) {
+            expect(getCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+        }
+    });
+
+    it.each(types)("should store a key with the cache callback (if set)",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(shouldCache ? 1 : 0);
+        if (shouldCache) {
+            expect(storeCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+            expect(storeCrossSigningKeyCache.mock.calls[0][1]).toBe(testKey);
+        }
+    });
+
+    it.each(types)("does not store a bad key to the cache",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(badKey);
+        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { storeCrossSigningKeyCache },
+        );
+        await expect(info.getCrossSigningKey(type, masterKeyPub)).rejects.toThrow();
+        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(0);
+    });
+
+    it.each(types)("does not store a value to the cache if it came from the cache",
+        async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockImplementation(() => {
+            if (shouldCache) {
+                return Promise.reject(new Error("Regular callback called"));
+            } else {
+                return Promise.resolve(testKey);
+            }
+        });
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+        const storeCrossSigningKeyCache = jest.fn().mockRejectedValue(
+            new Error("Tried to store a value from cache"),
+        );
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(0);
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+    });
+
+    it.each(types)("requests a key from the cache callback (if set) and then calls app" +
+        " if one is not found", async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const storeCrossSigningKeyCache = jest.fn();
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKey.mock.calls.length).toBe(1);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+
+        /* Also expect that the cache gets updated */
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+    });
+
+    it.each(types)("requests a key from the cache callback (if set) and then" +
+        " calls app if that key doesn't match", async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(badKey);
+        const storeCrossSigningKeyCache = jest.fn();
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKey.mock.calls.length).toBe(1);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+
+        /* Also expect that the cache gets updated */
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+    });
+});
+
+/*
+ * Note that MemoryStore is weird.  It's only used for testing - as far as I can tell,
+ * it's not possible to get one in normal execution unless you hack as we do here.
+ */
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("CrossSigning > createCryptoStoreCacheCallbacks [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(() => {
+        store = dbFactory();
+    });
+
+    beforeEach(async () => {
+        await store.deleteAllData();
+    });
+
+    it("should cache data to the store and retrieve it", async () => {
+        await store.startup();
+        const olmDevice = new OlmDevice(store);
+        const { getCrossSigningKeyCache, storeCrossSigningKeyCache } =
+              createCryptoStoreCacheCallbacks(store, olmDevice);
+        await storeCrossSigningKeyCache("self_signing", testKey);
+
+        // If we've not saved anything, don't expect anything
+        // Definitely don't accidentally return the wrong key for the type
+        const nokey = await getCrossSigningKeyCache("self", "");
+        expect(nokey).toBeNull();
+
+        const key = await getCrossSigningKeyCache("self_signing", "");
+        expect(new Uint8Array(key)).toEqual(testKey);
+    });
+});

spec/unit/crypto/DeviceList.spec.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
 Copyright 2018, 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,14 +16,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import DeviceList from '../../../lib/crypto/DeviceList';
-import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
-import testUtils from '../../test-utils';
-import utils from '../../../lib/utils';
-import logger from '../../../src/logger';
-
-import expect from 'expect';
-import Promise from 'bluebird';
+import {logger} from "../../../src/logger";
+import * as utils from "../../../src/utils";
+import {MemoryCryptoStore} from "../../../src/crypto/store/memory-crypto-store";
+import {DeviceList} from "../../../src/crypto/DeviceList";
 
 const signedDeviceList = {
     "failures": {},
@@ -54,17 +51,45 @@ const signedDeviceList = {
     },
 };
 
+const signedDeviceList2 = {
+    "failures": {},
+    "device_keys": {
+        "@test2:sw1v.org": {
+            "QJVRHWAKGH": {
+                "signatures": {
+                    "@test2:sw1v.org": {
+                        "ed25519:QJVRHWAKGH":
+                            "w1xxdLe1iIqzEFHLRVYQeuiM6t2N2ZRiI8s5nDKxf054BP8" +
+                            "1CPEX/AQXh5BhkKAVMlKnwg4T9zU1/wBALeajk3",
+                    },
+                },
+                "user_id": "@test2:sw1v.org",
+                "keys": {
+                    "ed25519:QJVRHWAKGH":
+                        "Ig0/C6T+bBII1l2By2Wnnvtjp1nm/iXBlLU5/QESFXL",
+                    "curve25519:QJVRHWAKGH":
+                        "YR3eQnUvTQzGlWih4rsmJkKxpDxzgkgIgsBd1DEZIbm",
+                },
+                "algorithms": [
+                    "m.olm.v1.curve25519-aes-sha2",
+                    "m.megolm.v1.aes-sha2",
+                ],
+                "device_id": "QJVRHWAKGH",
+                "unsigned": {},
+            },
+        },
+    },
+};
+
 describe('DeviceList', function() {
     let downloadSpy;
     let cryptoStore;
     let deviceLists = [];
 
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-
         deviceLists = [];
 
-        downloadSpy = expect.createSpy();
+        downloadSpy = jest.fn();
         cryptoStore = new MemoryCryptoStore();
     });
 
@@ -74,14 +99,16 @@ describe('DeviceList', function() {
         }
     });
 
-    function createTestDeviceList() {
+    function createTestDeviceList(keyDownloadChunkSize = 250) {
         const baseApis = {
             downloadKeysForUsers: downloadSpy,
+            getUserId: () => '@test1:sw1v.org',
+            deviceId: 'HGKAWHRVJQ',
         };
         const mockOlm = {
             verifySignature: function(key, message, signature) {},
         };
-        const dl = new DeviceList(baseApis, cryptoStore, mockOlm);
+        const dl = new DeviceList(baseApis, cryptoStore, mockOlm, keyDownloadChunkSize);
         deviceLists.push(dl);
         return dl;
     }
@@ -91,8 +118,8 @@ describe('DeviceList', function() {
 
         dl.startTrackingDeviceList('@test1:sw1v.org');
 
-        const queryDefer1 = Promise.defer();
-        downloadSpy.andReturn(queryDefer1.promise);
+        const queryDefer1 = utils.defer();
+        downloadSpy.mockReturnValue(queryDefer1.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
         expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
@@ -110,16 +137,16 @@ describe('DeviceList', function() {
 
         dl.startTrackingDeviceList('@test1:sw1v.org');
 
-        const queryDefer1 = Promise.defer();
-        downloadSpy.andReturn(queryDefer1.promise);
+        const queryDefer1 = utils.defer();
+        downloadSpy.mockReturnValue(queryDefer1.promise);
 
         const prom1 = dl.refreshOutdatedDeviceLists();
         expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
-        downloadSpy.reset();
+        downloadSpy.mockReset();
 
         // outdated notif arrives while the request is in flight.
-        const queryDefer2 = Promise.defer();
-        downloadSpy.andReturn(queryDefer2.promise);
+        const queryDefer2 = utils.defer();
+        downloadSpy.mockReturnValue(queryDefer2.promise);
 
         dl.invalidateUserDeviceList('@test1:sw1v.org');
         dl.refreshOutdatedDeviceLists();
@@ -136,10 +163,10 @@ describe('DeviceList', function() {
             // uh-oh; user restarts before second request completes. The new instance
             // should know we never got a complete device list.
             logger.log("Creating new devicelist to simulate app reload");
-            downloadSpy.reset();
+            downloadSpy.mockReset();
             const dl2 = createTestDeviceList();
-            const queryDefer3 = Promise.defer();
-            downloadSpy.andReturn(queryDefer3.promise);
+            const queryDefer3 = utils.defer();
+            downloadSpy.mockReturnValue(queryDefer3.promise);
 
             const prom3 = dl2.refreshOutdatedDeviceLists();
             expect(downloadSpy).toHaveBeenCalledWith(['@test1:sw1v.org'], {});
@@ -153,4 +180,30 @@ describe('DeviceList', function() {
             expect(Object.keys(storedKeys)).toEqual(['HGKAWHRVJQ']);
         });
     });
+
+    it("should download device keys in batches", function() {
+        const dl = createTestDeviceList(1);
+
+        dl.startTrackingDeviceList('@test1:sw1v.org');
+        dl.startTrackingDeviceList('@test2:sw1v.org');
+
+        const queryDefer1 = utils.defer();
+        downloadSpy.mockReturnValueOnce(queryDefer1.promise);
+        const queryDefer2 = utils.defer();
+        downloadSpy.mockReturnValueOnce(queryDefer2.promise);
+
+        const prom1 = dl.refreshOutdatedDeviceLists();
+        expect(downloadSpy).toBeCalledTimes(2);
+        expect(downloadSpy).toHaveBeenNthCalledWith(1, ['@test1:sw1v.org'], {});
+        expect(downloadSpy).toHaveBeenNthCalledWith(2, ['@test2:sw1v.org'], {});
+        queryDefer1.resolve(utils.deepCopy(signedDeviceList));
+        queryDefer2.resolve(utils.deepCopy(signedDeviceList2));
+
+        return prom1.then(() => {
+            const storedKeys1 = dl.getRawStoredDevicesForUser('@test1:sw1v.org');
+            expect(Object.keys(storedKeys1)).toEqual(['HGKAWHRVJQ']);
+            const storedKeys2 = dl.getRawStoredDevicesForUser('@test2:sw1v.org');
+            expect(Object.keys(storedKeys2)).toEqual(['QJVRHWAKGH']);
+        });
+    });
 });

spec/unit/crypto/algorithms/megolm.spec.js

@@ -1,18 +1,16 @@
 import '../../../olm-loader';
+import * as algorithms from "../../../../src/crypto/algorithms";
+import {MemoryCryptoStore} from "../../../../src/crypto/store/memory-crypto-store";
+import {MockStorageApi} from "../../../MockStorageApi";
+import * as testUtils from "../../../test-utils";
+import {OlmDevice} from "../../../../src/crypto/OlmDevice";
+import {Crypto} from "../../../../src/crypto";
+import {logger} from "../../../../src/logger";
+import {MatrixEvent} from "../../../../src/models/event";
+import {TestClient} from "../../../TestClient";
+import {Room} from "../../../../src/models/room";
+import * as olmlib from "../../../../src/crypto/olmlib";
 
-import expect from 'expect';
-import Promise from 'bluebird';
-
-import sdk from '../../../..';
-import algorithms from '../../../../lib/crypto/algorithms';
-import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
-import MockStorageApi from '../../../MockStorageApi';
-import testUtils from '../../../test-utils';
-import OlmDevice from '../../../../lib/crypto/OlmDevice';
-import Crypto from '../../../../lib/crypto';
-import logger from '../../../../src/logger';
-
-const MatrixEvent = sdk.MatrixEvent;
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 const MegolmEncryption = algorithms.ENCRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 
@@ -26,16 +24,16 @@ describe("MegolmDecryption", function() {
         return;
     }
 
+    beforeAll(function() {
+        return Olm.init();
+    });
+
     let megolmDecryption;
     let mockOlmLib;
     let mockCrypto;
     let mockBaseApis;
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-
-        await Olm.init();
-
         mockCrypto = testUtils.mock(Crypto, 'Crypto');
         mockBaseApis = {};
 
@@ -55,9 +53,9 @@ describe("MegolmDecryption", function() {
 
         // we stub out the olm encryption bits
         mockOlmLib = {};
-        mockOlmLib.ensureOlmSessionsForDevices = expect.createSpy();
+        mockOlmLib.ensureOlmSessionsForDevices = jest.fn();
         mockOlmLib.encryptMessageForDevice =
-            expect.createSpy().andReturn(Promise.resolve());
+            jest.fn().mockResolvedValue(undefined);
         megolmDecryption.olmlib = mockOlmLib;
     });
 
@@ -135,22 +133,22 @@ describe("MegolmDecryption", function() {
 
                 // set up some pre-conditions for the share call
                 const deviceInfo = {};
-                mockCrypto.getStoredDevice.andReturn(deviceInfo);
+                mockCrypto.getStoredDevice.mockReturnValue(deviceInfo);
 
-                mockOlmLib.ensureOlmSessionsForDevices.andReturn(
-                    Promise.resolve({'@alice:foo': {'alidevice': {
+                mockOlmLib.ensureOlmSessionsForDevices.mockResolvedValue({
+                    '@alice:foo': {'alidevice': {
                         sessionId: 'alisession',
-                    }}}),
-                );
+                    }},
+                });
 
                 const awaitEncryptForDevice = new Promise((res, rej) => {
-                    mockOlmLib.encryptMessageForDevice.andCall(() => {
+                    mockOlmLib.encryptMessageForDevice.mockImplementation(() => {
                         res();
                         return Promise.resolve();
                     });
                 });
 
-                mockBaseApis.sendToDevice = expect.createSpy();
+                mockBaseApis.sendToDevice = jest.fn();
 
                 // do the share
                 megolmDecryption.shareKeysWithDevice(keyRequest);
@@ -160,21 +158,20 @@ describe("MegolmDecryption", function() {
             }).then(() => {
                 // check that it called encryptMessageForDevice with
                 // appropriate args.
-                expect(mockOlmLib.encryptMessageForDevice.calls.length)
-                    .toEqual(1);
+                expect(mockOlmLib.encryptMessageForDevice).toBeCalledTimes(1);
 
-                const call = mockOlmLib.encryptMessageForDevice.calls[0];
-                const payload = call.arguments[6];
+                const call = mockOlmLib.encryptMessageForDevice.mock.calls[0];
+                const payload = call[6];
 
                 expect(payload.type).toEqual("m.forwarded_room_key");
-                expect(payload.content).toInclude({
+                expect(payload.content).toMatchObject({
                     sender_key: "SENDER_CURVE25519",
                     sender_claimed_ed25519_key: "SENDER_ED25519",
                     session_id: groupSession.session_id(),
                     chain_index: 0,
                     forwarding_curve25519_key_chain: [],
                 });
-                expect(payload.content.session_key).toExist();
+                expect(payload.content.session_key).toBeDefined();
             });
         });
 
@@ -201,13 +198,12 @@ describe("MegolmDecryption", function() {
                 origin_server_ts: 1507753886000,
             });
 
-            const successHandler = expect.createSpy();
-            const failureHandler = expect.createSpy()
-                .andCall((err) => {
-                    expect(err.toString()).toMatch(
-                        /Duplicate message index, possible replay attack/,
-                    );
-                });
+            const successHandler = jest.fn();
+            const failureHandler = jest.fn((err) => {
+                expect(err.toString()).toMatch(
+                    /Duplicate message index, possible replay attack/,
+                );
+            });
 
             return megolmDecryption.decryptEvent(event1).then((res) => {
                 const event2 = new MatrixEvent({
@@ -228,7 +224,7 @@ describe("MegolmDecryption", function() {
                 successHandler,
                 failureHandler,
             ).then(() => {
-                expect(successHandler).toNotHaveBeenCalled();
+                expect(successHandler).not.toHaveBeenCalled();
                 expect(failureHandler).toHaveBeenCalled();
             });
         });
@@ -266,10 +262,10 @@ describe("MegolmDecryption", function() {
             const cryptoStore = new MemoryCryptoStore(mockStorage);
 
             const olmDevice = new OlmDevice(cryptoStore);
-            olmDevice.verifySignature = expect.createSpy();
+            olmDevice.verifySignature = jest.fn();
             await olmDevice.init();
 
-            mockBaseApis.claimOneTimeKeys = expect.createSpy().andReturn(Promise.resolve({
+            mockBaseApis.claimOneTimeKeys = jest.fn().mockReturnValue(Promise.resolve({
                 one_time_keys: {
                     '@alice:home.server': {
                         aliceDevice: {
@@ -285,22 +281,26 @@ describe("MegolmDecryption", function() {
                     },
                 },
             }));
-            mockBaseApis.sendToDevice = expect.createSpy().andReturn(Promise.resolve());
+            mockBaseApis.sendToDevice = jest.fn().mockResolvedValue(undefined);
 
-            mockCrypto.downloadKeys.andReturn(Promise.resolve({
+            mockCrypto.downloadKeys.mockReturnValue(Promise.resolve({
                 '@alice:home.server': {
                     aliceDevice: {
                         deviceId: 'aliceDevice',
-                        isBlocked: expect.createSpy().andReturn(false),
-                        isUnverified: expect.createSpy().andReturn(false),
-                        getIdentityKey: expect.createSpy().andReturn(
+                        isBlocked: jest.fn().mockReturnValue(false),
+                        isUnverified: jest.fn().mockReturnValue(false),
+                        getIdentityKey: jest.fn().mockReturnValue(
                             'YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE',
                         ),
-                        getFingerprint: expect.createSpy().andReturn(''),
+                        getFingerprint: jest.fn().mockReturnValue(''),
                     },
                 },
             }));
 
+            mockCrypto.checkDeviceTrust.mockReturnValue({
+                isVerified: () => false,
+            });
+
             const megolmEncryption = new MegolmEncryption({
                 userId: '@user:id',
                 crypto: mockCrypto,
@@ -312,10 +312,10 @@ describe("MegolmDecryption", function() {
                 },
             });
             const mockRoom = {
-                getEncryptionTargetMembers: expect.createSpy().andReturn(
+                getEncryptionTargetMembers: jest.fn().mockReturnValue(
                     [{userId: "@alice:home.server"}],
                 ),
-                getBlacklistUnverifiedDevices: expect.createSpy().andReturn(false),
+                getBlacklistUnverifiedDevices: jest.fn().mockReturnValue(false),
             };
             const ct1 = await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
                 body: "Some text",
@@ -323,28 +323,372 @@ describe("MegolmDecryption", function() {
             expect(mockRoom.getEncryptionTargetMembers).toHaveBeenCalled();
 
             // this should have claimed a key for alice as it's starting a new session
-            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalled(
-                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
             );
             expect(mockCrypto.downloadKeys).toHaveBeenCalledWith(
                 ['@alice:home.server'], false,
             );
             expect(mockBaseApis.sendToDevice).toHaveBeenCalled();
-            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalled(
-                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+            expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
             );
 
-            mockBaseApis.claimOneTimeKeys.reset();
+            mockBaseApis.claimOneTimeKeys.mockReset();
 
             const ct2 = await megolmEncryption.encryptMessage(mockRoom, "a.fake.type", {
                 body: "Some more text",
             });
 
             // this should *not* have claimed a key as it should be using the same session
-            expect(mockBaseApis.claimOneTimeKeys).toNotHaveBeenCalled();
+            expect(mockBaseApis.claimOneTimeKeys).not.toHaveBeenCalled();
 
             // likewise they should show the same session ID
             expect(ct2.session_id).toEqual(ct1.session_id);
         });
     });
+
+    it("notifies devices that have been blocked", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient1 = (new TestClient(
+            "@bob:example.com", "bobdevice1",
+        )).client;
+        const bobClient2 = (new TestClient(
+            "@bob:example.com", "bobdevice2",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient1.initCrypto(),
+            bobClient2.initCrypto(),
+        ]);
+        const aliceDevice = aliceClient._crypto._olmDevice;
+        const bobDevice1 = bobClient1._crypto._olmDevice;
+        const bobDevice2 = bobClient2._crypto._olmDevice;
+
+        const encryptionCfg = {
+            "algorithm": "m.megolm.v1.aes-sha2",
+        };
+        const roomId = "!someroom";
+        const room = new Room(roomId, aliceClient, "@alice:example.com", {});
+        room.getEncryptionTargetMembers = async function() {
+            return [{userId: "@bob:example.com"}];
+        };
+        room.setBlacklistUnverifiedDevices(true);
+        aliceClient.store.storeRoom(room);
+        await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+
+        const BOB_DEVICES = {
+            bobdevice1: {
+                user_id: "@bob:example.com",
+                device_id: "bobdevice1",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:Dynabook": bobDevice1.deviceEd25519Key,
+                    "curve25519:Dynabook": bobDevice1.deviceCurve25519Key,
+                },
+                verified: 0,
+            },
+            bobdevice2: {
+                user_id: "@bob:example.com",
+                device_id: "bobdevice2",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:Dynabook": bobDevice2.deviceEd25519Key,
+                    "curve25519:Dynabook": bobDevice2.deviceCurve25519Key,
+                },
+                verified: -1,
+            },
+        };
+
+        aliceClient._crypto._deviceList.storeDevicesForUser(
+            "@bob:example.com", BOB_DEVICES,
+        );
+        aliceClient._crypto._deviceList.downloadKeys = async function(userIds) {
+            return this._getDevicesFromStore(userIds);
+        };
+
+        let run = false;
+        aliceClient.sendToDevice = async (msgtype, contentMap) => {
+            run = true;
+            expect(msgtype).toBe("org.matrix.room_key.withheld");
+            delete contentMap["@bob:example.com"].bobdevice1.session_id;
+            delete contentMap["@bob:example.com"].bobdevice2.session_id;
+            expect(contentMap).toStrictEqual({
+                '@bob:example.com': {
+                    bobdevice1: {
+                        algorithm: "m.megolm.v1.aes-sha2",
+                        room_id: roomId,
+                        code: 'm.unverified',
+                        reason:
+                        'The sender has disabled encrypting to unverified devices.',
+                        sender_key: aliceDevice.deviceCurve25519Key,
+                    },
+                    bobdevice2: {
+                        algorithm: "m.megolm.v1.aes-sha2",
+                        room_id: roomId,
+                        code: 'm.blacklisted',
+                        reason: 'The sender has blocked you.',
+                        sender_key: aliceDevice.deviceCurve25519Key,
+                    },
+                },
+            });
+        };
+
+        const event = new MatrixEvent({
+            type: "m.room.message",
+            sender: "@alice:example.com",
+            room_id: roomId,
+            event_id: "$event",
+            content: {
+                msgtype: "m.text",
+                body: "secret",
+            },
+        });
+        await aliceClient._crypto.encryptEvent(event, room);
+
+        expect(run).toBe(true);
+
+        aliceClient.stopClient();
+        bobClient1.stopClient();
+        bobClient2.stopClient();
+    });
+
+    it("notifies devices when unable to create olm session", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        const aliceDevice = aliceClient._crypto._olmDevice;
+        const bobDevice = bobClient._crypto._olmDevice;
+
+        const encryptionCfg = {
+            "algorithm": "m.megolm.v1.aes-sha2",
+        };
+        const roomId = "!someroom";
+        const aliceRoom = new Room(roomId, aliceClient, "@alice:example.com", {});
+        const bobRoom = new Room(roomId, bobClient, "@bob:example.com", {});
+        aliceClient.store.storeRoom(aliceRoom);
+        bobClient.store.storeRoom(bobRoom);
+        await aliceClient.setRoomEncryption(roomId, encryptionCfg);
+        await bobClient.setRoomEncryption(roomId, encryptionCfg);
+
+        aliceRoom.getEncryptionTargetMembers = async () => {
+            return [
+                {
+                    userId: "@alice:example.com",
+                    membership: "join",
+                },
+                {
+                    userId: "@bob:example.com",
+                    membership: "join",
+                },
+            ];
+        };
+        const BOB_DEVICES = {
+            bobdevice: {
+                user_id: "@bob:example.com",
+                device_id: "bobdevice",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:bobdevice": bobDevice.deviceEd25519Key,
+                    "curve25519:bobdevice": bobDevice.deviceCurve25519Key,
+                },
+                known: true,
+                verified: 1,
+            },
+        };
+
+        aliceClient._crypto._deviceList.storeDevicesForUser(
+            "@bob:example.com", BOB_DEVICES,
+        );
+        aliceClient._crypto._deviceList.downloadKeys = async function(userIds) {
+            return this._getDevicesFromStore(userIds);
+        };
+
+        aliceClient.claimOneTimeKeys = async () => {
+            // Bob has no one-time keys
+            return {
+                one_time_keys: {},
+            };
+        };
+
+        const sendPromise = new Promise((resolve, reject) => {
+            aliceClient.sendToDevice = async (msgtype, contentMap) => {
+                expect(msgtype).toBe("org.matrix.room_key.withheld");
+                expect(contentMap).toStrictEqual({
+                    '@bob:example.com': {
+                        bobdevice: {
+                            algorithm: "m.megolm.v1.aes-sha2",
+                            code: 'm.no_olm',
+                            reason: 'Unable to establish a secure channel.',
+                            sender_key: aliceDevice.deviceCurve25519Key,
+                        },
+                    },
+                });
+                resolve();
+            };
+        });
+
+        const event = new MatrixEvent({
+            type: "m.room.message",
+            sender: "@alice:example.com",
+            room_id: roomId,
+            event_id: "$event",
+            content: {},
+        });
+        await aliceClient._crypto.encryptEvent(event, aliceRoom);
+        await sendPromise;
+    });
+
+    it("throws an error describing why it doesn't have a key", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        const bobDevice = bobClient._crypto._olmDevice;
+
+        const roomId = "!someroom";
+
+        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+            type: "org.matrix.room_key.withheld",
+            sender: "@bob:example.com",
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                room_id: roomId,
+                session_id: "session_id",
+                sender_key: bobDevice.deviceCurve25519Key,
+                code: "m.blacklisted",
+                reason: "You have been blocked",
+            },
+        }));
+
+        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            event_id: "$event",
+            room_id: roomId,
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                ciphertext: "blablabla",
+                device_id: "bobdevice",
+                sender_key: bobDevice.deviceCurve25519Key,
+                session_id: "session_id",
+            },
+        }))).rejects.toThrow("The sender has blocked you.");
+    });
+
+    it("throws an error describing the lack of an olm session", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        aliceClient._crypto.downloadKeys = async () => {};
+        const bobDevice = bobClient._crypto._olmDevice;
+
+        const roomId = "!someroom";
+
+        const now = Date.now();
+
+        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+            type: "org.matrix.room_key.withheld",
+            sender: "@bob:example.com",
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                room_id: roomId,
+                session_id: "session_id",
+                sender_key: bobDevice.deviceCurve25519Key,
+                code: "m.no_olm",
+                reason: "Unable to establish a secure channel.",
+            },
+        }));
+
+        await new Promise((resolve) => {
+            setTimeout(resolve, 100);
+        });
+
+        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            event_id: "$event",
+            room_id: roomId,
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                ciphertext: "blablabla",
+                device_id: "bobdevice",
+                sender_key: bobDevice.deviceCurve25519Key,
+                session_id: "session_id",
+            },
+            origin_server_ts: now,
+        }))).rejects.toThrow("The sender was unable to establish a secure channel.");
+    });
+
+    it("throws an error to indicate a wedged olm session", async function() {
+        const aliceClient = (new TestClient(
+            "@alice:example.com", "alicedevice",
+        )).client;
+        const bobClient = (new TestClient(
+            "@bob:example.com", "bobdevice",
+        )).client;
+        await Promise.all([
+            aliceClient.initCrypto(),
+            bobClient.initCrypto(),
+        ]);
+        const bobDevice = bobClient._crypto._olmDevice;
+        aliceClient._crypto.downloadKeys = async () => {};
+
+        const roomId = "!someroom";
+
+        const now = Date.now();
+
+        // pretend we got an event that we can't decrypt
+        aliceClient._crypto._onToDeviceEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            content: {
+                msgtype: "m.bad.encrypted",
+                algorithm: "m.megolm.v1.aes-sha2",
+                session_id: "session_id",
+                sender_key: bobDevice.deviceCurve25519Key,
+            },
+        }));
+
+        await new Promise((resolve) => {
+            setTimeout(resolve, 100);
+        });
+
+        await expect(aliceClient._crypto.decryptEvent(new MatrixEvent({
+            type: "m.room.encrypted",
+            sender: "@bob:example.com",
+            event_id: "$event",
+            room_id: roomId,
+            content: {
+                algorithm: "m.megolm.v1.aes-sha2",
+                ciphertext: "blablabla",
+                device_id: "bobdevice",
+                sender_key: bobDevice.deviceCurve25519Key,
+                session_id: "session_id",
+            },
+            origin_server_ts: now,
+        }))).rejects.toThrow("The secure channel with the sender was corrupted.");
+    });
 });

spec/unit/crypto/algorithms/olm.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018,2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,16 +16,12 @@ limitations under the License.
 */
 
 import '../../../olm-loader';
-
-import expect from 'expect';
-import MemoryCryptoStore from '../../../../lib/crypto/store/memory-crypto-store.js';
-import MockStorageApi from '../../../MockStorageApi';
-import testUtils from '../../../test-utils';
-import logger from '../../../../src/logger';
-
-import OlmDevice from '../../../../lib/crypto/OlmDevice';
-import olmlib from '../../../../lib/crypto/olmlib';
-import DeviceInfo from '../../../../lib/crypto/deviceinfo';
+import {MemoryCryptoStore} from "../../../../src/crypto/store/memory-crypto-store";
+import {MockStorageApi} from "../../../MockStorageApi";
+import {logger} from "../../../../src/logger";
+import {OlmDevice} from "../../../../src/crypto/OlmDevice";
+import * as olmlib from "../../../../src/crypto/olmlib";
+import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
 
 function makeOlmDevice() {
     const mockStorage = new MockStorageApi();
@@ -44,20 +41,20 @@ async function setupSession(initiator, opponent) {
     return sid;
 }
 
-describe("OlmDecryption", function() {
+describe("OlmDevice", function() {
     if (!global.Olm) {
         logger.warn('Not running megolm unit tests: libolm not present');
         return;
     }
 
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
     let aliceOlmDevice;
     let bobOlmDevice;
 
     beforeEach(async function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-
-        await global.Olm.init();
-
         aliceOlmDevice = makeOlmDevice();
         bobOlmDevice = makeOlmDevice();
         await aliceOlmDevice.init();
@@ -84,6 +81,60 @@ describe("OlmDecryption", function() {
             );
         });
 
+        it('exports picked account and olm sessions', async function() {
+            const sessionId = await setupSession(aliceOlmDevice, bobOlmDevice);
+
+            const exported = await bobOlmDevice.export();
+            // At this moment only Alice (the “initiator” in setupSession) has a session
+            expect(exported.sessions).toEqual([]);
+
+            const MESSAGE = (
+                "The olm or proteus is an aquatic salamander"
+                + " in the family Proteidae"
+            );
+            const ciphertext = await aliceOlmDevice.encryptMessage(
+                bobOlmDevice.deviceCurve25519Key,
+                sessionId,
+                MESSAGE,
+            );
+
+            const bobRecreatedOlmDevice = makeOlmDevice();
+            bobRecreatedOlmDevice.init({ fromExportedDevice: exported });
+
+            const decrypted = await bobRecreatedOlmDevice.createInboundSession(
+                aliceOlmDevice.deviceCurve25519Key,
+                ciphertext.type,
+                ciphertext.body,
+            );
+            expect(decrypted.payload).toEqual(MESSAGE);
+
+            const exportedAgain = await bobRecreatedOlmDevice.export();
+            // this time we expect Bob to have a session to export
+            expect(exportedAgain.sessions).toHaveLength(1);
+
+            const MESSAGE_2 = (
+                "In contrast to most amphibians,"
+                + " the olm is entirely aquatic"
+            );
+            const ciphertext2 = await aliceOlmDevice.encryptMessage(
+                bobOlmDevice.deviceCurve25519Key,
+                sessionId,
+                MESSAGE_2,
+            );
+
+            const bobRecreatedAgainOlmDevice = makeOlmDevice();
+            bobRecreatedAgainOlmDevice.init({ fromExportedDevice: exportedAgain });
+
+            // Note: "decrypted_2" does not have the same structure as "decrypted"
+            const decrypted2 = await bobRecreatedAgainOlmDevice.decryptMessage(
+                aliceOlmDevice.deviceCurve25519Key,
+                decrypted.session_id,
+                ciphertext2.type,
+                ciphertext2.body,
+            );
+            expect(decrypted2).toEqual(MESSAGE_2);
+        });
+
         it("creates only one session at a time", async function() {
             // if we call ensureOlmSessionsForDevices multiple times, it should
             // only try to create one session at a time, even if the server is
@@ -139,5 +190,91 @@ describe("OlmDecryption", function() {
             // new session and will have called claimOneTimeKeys
             expect(count).toBe(2);
         });
+
+        it("avoids deadlocks when two tasks are ensuring the same devices", async function() {
+            // This test checks whether `ensureOlmSessionsForDevices` properly
+            // handles multiple tasks in flight ensuring some set of devices in
+            // common without deadlocks.
+
+            let claimRequestCount = 0;
+            const baseApis = {
+                claimOneTimeKeys: () => {
+                    // simulate a very slow server (.5 seconds to respond)
+                    claimRequestCount++;
+                    return new Promise((resolve, reject) => {
+                        setTimeout(reject, 500);
+                    });
+                },
+            };
+
+            const deviceBobA = DeviceInfo.fromStorage({
+                keys: {
+                    "curve25519:BOB-A": "akey",
+                },
+            }, "BOB-A");
+            const deviceBobB = DeviceInfo.fromStorage({
+                keys: {
+                    "curve25519:BOB-B": "bkey",
+                },
+            }, "BOB-B");
+
+            // There's no required ordering of devices per user, so here we
+            // create two different orderings so that each task reserves a
+            // device the other task needs before continuing.
+            const devicesByUserAB = {
+                "@bob:example.com": [
+                    deviceBobA,
+                    deviceBobB,
+                ],
+            };
+            const devicesByUserBA = {
+                "@bob:example.com": [
+                    deviceBobB,
+                    deviceBobA,
+                ],
+            };
+
+            function alwaysSucceed(promise) {
+                // swallow any exception thrown by a promise, so that
+                // Promise.all doesn't abort
+                return promise.catch(() => {});
+            }
+
+            const task1 = alwaysSucceed(olmlib.ensureOlmSessionsForDevices(
+                aliceOlmDevice, baseApis, devicesByUserAB,
+            ));
+
+            // After a single tick through the first task, it should have
+            // claimed ownership of all devices to avoid deadlocking others.
+            expect(Object.keys(aliceOlmDevice._sessionsInProgress).length).toBe(2);
+
+            const task2 = alwaysSucceed(olmlib.ensureOlmSessionsForDevices(
+                aliceOlmDevice, baseApis, devicesByUserBA,
+            ));
+
+            // The second task should not have changed the ownership count, as
+            // it's waiting on the first task.
+            expect(Object.keys(aliceOlmDevice._sessionsInProgress).length).toBe(2);
+
+            // Track the tasks, but don't await them yet.
+            const promises = Promise.all([
+                task1,
+                task2,
+            ]);
+
+            await new Promise((resolve) => {
+                setTimeout(resolve, 200);
+            });
+
+            // After .2s, the first task should have made an initial claim request.
+            expect(claimRequestCount).toBe(1);
+
+            await promises;
+
+            // After waiting for both tasks to complete, the first task should
+            // have failed, so the second task should have tried to create a
+            // new session and will have called claimOneTimeKeys
+            expect(claimRequestCount).toBe(2);
+        });
     });
 });

spec/unit/crypto/backup.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,25 +16,21 @@ limitations under the License.
 */
 
 import '../../olm-loader';
-
-import expect from 'expect';
-import Promise from 'bluebird';
-
-import sdk from '../../..';
-import algorithms from '../../../lib/crypto/algorithms';
-import WebStorageSessionStore from '../../../lib/store/session/webstorage';
-import MemoryCryptoStore from '../../../lib/crypto/store/memory-crypto-store.js';
-import MockStorageApi from '../../MockStorageApi';
-import testUtils from '../../test-utils';
-
-import OlmDevice from '../../../lib/crypto/OlmDevice';
-import Crypto from '../../../lib/crypto';
-import logger from '../../../src/logger';
+import {logger} from "../../../src/logger";
+import * as olmlib from "../../../src/crypto/olmlib";
+import {MatrixClient} from "../../../src/client";
+import {MatrixEvent} from "../../../src/models/event";
+import * as algorithms from "../../../src/crypto/algorithms";
+import {WebStorageSessionStore} from "../../../src/store/session/webstorage";
+import {MemoryCryptoStore} from "../../../src/crypto/store/memory-crypto-store";
+import {MockStorageApi} from "../../MockStorageApi";
+import * as testUtils from "../../test-utils";
+import {OlmDevice} from "../../../src/crypto/OlmDevice";
+import {Crypto} from "../../../src/crypto";
+import {resetCrossSigningKeys} from "./crypto-utils";
 
 const Olm = global.Olm;
 
-const MatrixClient = sdk.MatrixClient;
-const MatrixEvent = sdk.MatrixEvent;
 const MegolmDecryption = algorithms.DECRYPTION_CLASSES['m.megolm.v1.aes-sha2'];
 
 const ROOM_ID = '!ROOM:ID';
@@ -83,20 +80,30 @@ const BACKUP_INFO = {
     },
 };
 
+const keys = {};
+
+function getCrossSigningKey(type) {
+    return keys[type];
+}
+
+function saveCrossSigningKeys(k) {
+    Object.assign(keys, k);
+}
+
 function makeTestClient(sessionStore, cryptoStore) {
     const scheduler = [
         "getQueueForEvent", "queueEvent", "removeEventFromQueue",
         "setProcessFunction",
-    ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
+    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
     const store = [
         "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
         "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom",
         "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter",
         "storeFilter", "getSyncAccumulator", "startup", "deleteAllData",
-    ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
-    store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
-    store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
-    store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+    ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
+    store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null));
+    store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null));
+    store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null));
     return new MatrixClient({
         baseUrl: "https://my.home.server",
         idBaseUrl: "https://identity.server",
@@ -108,6 +115,7 @@ function makeTestClient(sessionStore, cryptoStore) {
         deviceId: "device",
         sessionStore: sessionStore,
         cryptoStore: cryptoStore,
+        cryptoCallbacks: { getCrossSigningKey, saveCrossSigningKeys },
     });
 }
 
@@ -117,6 +125,10 @@ describe("MegolmBackup", function() {
         return;
     }
 
+    beforeAll(function() {
+        return Olm.init();
+    });
+
     let olmDevice;
     let mockOlmLib;
     let mockCrypto;
@@ -125,9 +137,6 @@ describe("MegolmBackup", function() {
     let cryptoStore;
     let megolmDecryption;
     beforeEach(async function() {
-        await Olm.init();
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-
         mockCrypto = testUtils.mock(Crypto, 'Crypto');
         mockCrypto.backupKey = new Olm.PkEncryption();
         mockCrypto.backupKey.set_recipient_key(
@@ -143,9 +152,9 @@ describe("MegolmBackup", function() {
 
         // we stub out the olm encryption bits
         mockOlmLib = {};
-        mockOlmLib.ensureOlmSessionsForDevices = expect.createSpy();
+        mockOlmLib.ensureOlmSessionsForDevices = jest.fn();
         mockOlmLib.encryptMessageForDevice =
-            expect.createSpy().andReturn(Promise.resolve());
+            jest.fn().mockResolvedValue(undefined);
     });
 
     describe("backup", function() {
@@ -206,7 +215,7 @@ describe("MegolmBackup", function() {
             };
             mockCrypto.cancelRoomKeyRequest = function() {};
 
-            mockCrypto.backupGroupSession = expect.createSpy();
+            mockCrypto.backupGroupSession = jest.fn();
 
             return event.attemptDecryption(mockCrypto).then(() => {
                 return megolmDecryption.onRoomKeyEvent(event);
@@ -267,7 +276,7 @@ describe("MegolmBackup", function() {
                             callback, method, path, queryParams, data, opts,
                         ) {
                             ++numCalls;
-                            expect(numCalls).toBeLessThanOrEqualTo(1);
+                            expect(numCalls).toBeLessThanOrEqual(1);
                             if (numCalls >= 2) {
                                 // exit out of retry loop if there's something wrong
                                 reject(new Error("authedRequest called too many timmes"));
@@ -276,8 +285,8 @@ describe("MegolmBackup", function() {
                             expect(method).toBe("PUT");
                             expect(path).toBe("/room_keys/keys");
                             expect(queryParams.version).toBe(1);
-                            expect(data.rooms[ROOM_ID].sessions).toExist();
-                            expect(data.rooms[ROOM_ID].sessions).toIncludeKey(
+                            expect(data.rooms[ROOM_ID].sessions).toBeDefined();
+                            expect(data.rooms[ROOM_ID].sessions).toHaveProperty(
                                 groupSession.session_id(),
                             );
                             resolve();
@@ -296,6 +305,71 @@ describe("MegolmBackup", function() {
                 });
         });
 
+        it('signs backups with the cross-signing master key', async function() {
+            const groupSession = new Olm.OutboundGroupSession();
+            groupSession.create();
+            const ibGroupSession = new Olm.InboundGroupSession();
+            ibGroupSession.create(groupSession.session_key());
+
+            const client = makeTestClient(sessionStore, cryptoStore);
+
+            megolmDecryption = new MegolmDecryption({
+                userId: '@user:id',
+                crypto: mockCrypto,
+                olmDevice: olmDevice,
+                baseApis: client,
+                roomId: ROOM_ID,
+            });
+
+            megolmDecryption.olmlib = mockOlmLib;
+
+            await client.initCrypto();
+            let privateKeys;
+            client.uploadDeviceSigningKeys = async function(e) {return;};
+            client.uploadKeySignatures = async function(e) {return;};
+            client.on("crossSigning.saveCrossSigningKeys", function(e) {
+                privateKeys = e;
+            });
+            client.on("crossSigning.getKey", function(e) {
+                e.done(privateKeys[e.type]);
+            });
+            await resetCrossSigningKeys(client);
+            let numCalls = 0;
+            await new Promise((resolve, reject) => {
+                client._http.authedRequest = function(
+                    callback, method, path, queryParams, data, opts,
+                ) {
+                    ++numCalls;
+                    expect(numCalls).toBeLessThanOrEqual(1);
+                    if (numCalls >= 2) {
+                        // exit out of retry loop if there's something wrong
+                        reject(new Error("authedRequest called too many timmes"));
+                        return Promise.resolve({});
+                    }
+                    expect(method).toBe("POST");
+                    expect(path).toBe("/room_keys/version");
+                    try {
+                        // make sure auth_data is signed by the master key
+                        olmlib.pkVerify(
+                            data.auth_data, client.getCrossSigningId(), "@alice:bar",
+                        );
+                    } catch (e) {
+                        reject(e);
+                        return Promise.resolve({});
+                    }
+                    resolve();
+                    return Promise.resolve({});
+                };
+                client.createKeyBackupVersion({
+                    algorithm: "m.megolm_backup.v1",
+                    auth_data: {
+                        public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                    },
+                });
+            });
+            expect(numCalls).toBe(1);
+        });
+
         it('retries when a backup fails', function() {
             const groupSession = new Olm.OutboundGroupSession();
             groupSession.create();
@@ -305,16 +379,16 @@ describe("MegolmBackup", function() {
             const scheduler = [
                 "getQueueForEvent", "queueEvent", "removeEventFromQueue",
                 "setProcessFunction",
-            ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
+            ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
             const store = [
                 "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
                 "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom",
                 "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter",
                 "storeFilter", "getSyncAccumulator", "startup", "deleteAllData",
-            ].reduce((r, k) => {r[k] = expect.createSpy(); return r;}, {});
-            store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
-            store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
-            store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
+            ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {});
+            store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null));
+            store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null));
+            store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null));
             const client = new MatrixClient({
                 baseUrl: "https://my.home.server",
                 idBaseUrl: "https://identity.server",
@@ -372,7 +446,7 @@ describe("MegolmBackup", function() {
                             callback, method, path, queryParams, data, opts,
                         ) {
                             ++numCalls;
-                            expect(numCalls).toBeLessThanOrEqualTo(2);
+                            expect(numCalls).toBeLessThanOrEqual(2);
                             if (numCalls >= 3) {
                                 // exit out of retry loop if there's something wrong
                                 reject(new Error("authedRequest called too many timmes"));
@@ -381,8 +455,8 @@ describe("MegolmBackup", function() {
                             expect(method).toBe("PUT");
                             expect(path).toBe("/room_keys/keys");
                             expect(queryParams.version).toBe(1);
-                            expect(data.rooms[ROOM_ID].sessions).toExist();
-                            expect(data.rooms[ROOM_ID].sessions).toIncludeKey(
+                            expect(data.rooms[ROOM_ID].sessions).toBeDefined();
+                            expect(data.rooms[ROOM_ID].sessions).toHaveProperty(
                                 groupSession.session_id(),
                             );
                             if (numCalls > 1) {
@@ -444,6 +518,7 @@ describe("MegolmBackup", function() {
                 return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
             }).then((res) => {
                 expect(res.clearEvent.content).toEqual('testytest');
+                expect(res.untrusted).toBeTruthy(); // keys from backup are untrusted
             });
         });
 
@@ -468,5 +543,31 @@ describe("MegolmBackup", function() {
                 expect(res.clearEvent.content).toEqual('testytest');
             });
         });
+
+        it('has working cache functions', async function() {
+            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
+            await client._crypto.storeSessionBackupPrivateKey(key);
+            const result = await client._crypto.getSessionBackupPrivateKey();
+            expect(new Uint8Array(result)).toEqual(key);
+        });
+
+        it('caches session backup keys as it encounters them', async function() {
+            const cachedNull = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedNull).toBeNull();
+            client._http.authedRequest = function() {
+                return Promise.resolve(KEY_BACKUP_DATA);
+            };
+            await new Promise((resolve) => {
+                client.restoreKeyBackupWithRecoveryKey(
+                    "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                    ROOM_ID,
+                    SESSION_ID,
+                    BACKUP_INFO,
+                    { cacheCompleteCallback: resolve },
+                );
+            });
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedKey).not.toBeNull();
+        });
     });
 });

spec/unit/crypto/cross-signing.spec.js

@@ -0,0 +1,857 @@
+/*
+Copyright 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+import anotherjson from 'another-json';
+import * as olmlib from "../../../src/crypto/olmlib";
+import {TestClient} from '../../TestClient';
+import {HttpResponse, setHttpResponses} from '../../test-utils';
+import { resetCrossSigningKeys } from "./crypto-utils";
+import { MatrixError } from '../../../src/http-api';
+import {logger} from '../../../src/logger';
+
+async function makeTestClient(userInfo, options, keys) {
+    if (!keys) keys = {};
+
+    function getCrossSigningKey(type) {
+        return keys[type];
+    }
+
+    function saveCrossSigningKeys(k) {
+        Object.assign(keys, k);
+    }
+
+    if (!options) options = {};
+    options.cryptoCallbacks = Object.assign(
+        {}, { getCrossSigningKey, saveCrossSigningKeys }, options.cryptoCallbacks || {},
+    );
+    const client = (new TestClient(
+        userInfo.userId, userInfo.deviceId, undefined, undefined, options,
+    )).client;
+
+    await client.initCrypto();
+
+    return client;
+}
+
+describe("Cross Signing", function() {
+    if (!global.Olm) {
+        logger.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
+    it("should sign the master key with the device key", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = jest.fn(async (auth, keys) => {
+            await olmlib.verifySignature(
+                alice._crypto._olmDevice, keys.master_key, "@alice:example.com",
+                "Osborne2", alice._crypto._olmDevice.deviceEd25519Key,
+            );
+        });
+        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => {};
+        // set Alice's cross-signing key
+        await alice.bootstrapCrossSigning({
+            authUploadDeviceSigningKeys: async func => await func({}),
+        });
+        expect(alice.uploadDeviceSigningKeys).toHaveBeenCalled();
+    });
+
+    it("should abort bootstrap if device signing auth fails", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async (auth, keys) => {
+            const errorResponse = {
+                session: "sessionId",
+                flows: [
+                    {
+                        stages: [
+                            "m.login.password",
+                        ],
+                    },
+                ],
+                params: {},
+            };
+
+            // If we're not just polling for flows, add on error rejecting the
+            // auth attempt.
+            if (auth) {
+                Object.assign(errorResponse, {
+                    completed: [],
+                    error: "Invalid password",
+                    errcode: "M_FORBIDDEN",
+                });
+            }
+
+            const error = new MatrixError(errorResponse);
+            error.httpStatus == 401;
+            throw error;
+        };
+        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => { };
+        const authUploadDeviceSigningKeys = async func => await func({});
+
+        // Try bootstrap, expecting `authUploadDeviceSigningKeys` to pass
+        // through failure, stopping before actually applying changes.
+        let bootstrapDidThrow = false;
+        try {
+            await alice.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys,
+            });
+        } catch (e) {
+            if (e.errcode === "M_FORBIDDEN") {
+                bootstrapDidThrow = true;
+            }
+        }
+        expect(bootstrapDidThrow).toBeTruthy();
+    });
+
+    it("should upload a signature when a user is verified", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        // set Alice's cross-signing key
+        await resetCrossSigningKeys(alice);
+        // Alice downloads Bob's device key
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        "ed25519:bobs+master+pubkey": "bobs+master+pubkey",
+                    },
+                },
+            },
+        });
+        // Alice verifies Bob's key
+        const promise = new Promise((resolve, reject) => {
+            alice.uploadKeySignatures = (...args) => {
+                resolve(...args);
+            };
+        });
+        await alice.setDeviceVerified("@bob:example.com", "bobs+master+pubkey", true);
+        // Alice should send a signature of Bob's key to the server
+        await promise;
+    });
+
+    it("should get cross-signing keys from sync", async function() {
+        const masterKey = new Uint8Array([
+            0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+            0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+            0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+            0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+        ]);
+        const selfSigningKey = new Uint8Array([
+            0x1e, 0xf4, 0x01, 0x6d, 0x4f, 0xa1, 0x73, 0x66,
+            0x6b, 0xf8, 0x93, 0xf5, 0xb0, 0x4d, 0x17, 0xc0,
+            0x17, 0xb5, 0xa5, 0xf6, 0x59, 0x11, 0x8b, 0x49,
+            0x34, 0xf2, 0x4b, 0x64, 0x9b, 0x52, 0xf8, 0x5f,
+        ]);
+
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            {
+                cryptoCallbacks: {
+                    // will be called to sign our own device
+                    getCrossSigningKey: type => {
+                        if (type === 'master') {
+                            return masterKey;
+                        } else {
+                            return selfSigningKey;
+                        }
+                    },
+                },
+            },
+        );
+
+        const keyChangePromise = new Promise((resolve, reject) => {
+            alice.once("crossSigning.keysChanged", async (e) => {
+                resolve(e);
+                await alice.checkOwnCrossSigningTrust({
+                    allowPrivateKeyRequests: true,
+                });
+            });
+        });
+
+        const uploadSigsPromise = new Promise((resolve, reject) => {
+            alice.uploadKeySignatures = jest.fn(async (content) => {
+                await olmlib.verifySignature(
+                    alice._crypto._olmDevice,
+                    content["@alice:example.com"][
+                        "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk"
+                    ],
+                    "@alice:example.com",
+                    "Osborne2", alice._crypto._olmDevice.deviceEd25519Key,
+                );
+                olmlib.pkVerify(
+                    content["@alice:example.com"]["Osborne2"],
+                    "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
+                    "@alice:example.com",
+                );
+                resolve();
+            });
+        });
+
+        const deviceInfo = alice._crypto._deviceList._devices["@alice:example.com"]
+            .Osborne2;
+        const aliceDevice = {
+            user_id: "@alice:example.com",
+            device_id: "Osborne2",
+        };
+        aliceDevice.keys = deviceInfo.keys;
+        aliceDevice.algorithms = deviceInfo.algorithms;
+        await alice._crypto._signObject(aliceDevice);
+        olmlib.pkSign(aliceDevice, selfSigningKey, "@alice:example.com");
+
+        // feed sync result that includes master key, ssk, device key
+        const responses = [
+            HttpResponse.PUSH_RULES_RESPONSE,
+            {
+                method: "POST",
+                path: "/keys/upload",
+                data: {
+                    one_time_key_counts: {
+                        curve25519: 100,
+                        signed_curve25519: 100,
+                    },
+                },
+            },
+            HttpResponse.filterResponse("@alice:example.com"),
+            {
+                method: "GET",
+                path: "/sync",
+                data: {
+                    next_batch: "abcdefg",
+                    device_lists: {
+                        changed: [
+                            "@alice:example.com",
+                            "@bob:example.com",
+                        ],
+                    },
+                },
+            },
+            {
+                method: "POST",
+                path: "/keys/query",
+                data: {
+                    "failures": {},
+                    "device_keys": {
+                        "@alice:example.com": {
+                            "Osborne2": aliceDevice,
+                        },
+                    },
+                    "master_keys": {
+                        "@alice:example.com": {
+                            user_id: "@alice:example.com",
+                            usage: ["master"],
+                            keys: {
+                                "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk":
+                                "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk",
+                            },
+                        },
+                    },
+                    "self_signing_keys": {
+                        "@alice:example.com": {
+                            user_id: "@alice:example.com",
+                            usage: ["self-signing"],
+                            keys: {
+                                "ed25519:EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ":
+                                "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
+                            },
+                            signatures: {
+                                "@alice:example.com": {
+                                    "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk":
+                                    "Wqx/HXR851KIi8/u/UX+fbAMtq9Uj8sr8FsOcqrLfVYa6lAmbXs"
+                                    + "Vhfy4AlZ3dnEtjgZx0U0QDrghEn2eYBeOCA",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+            {
+                method: "POST",
+                path: "/keys/upload",
+                data: {
+                    one_time_key_counts: {
+                        curve25519: 100,
+                        signed_curve25519: 100,
+                    },
+                },
+            },
+        ];
+        setHttpResponses(alice, responses, true, true);
+
+        await alice.startClient();
+
+        // once ssk is confirmed, device key should be trusted
+        await keyChangePromise;
+        await uploadSigsPromise;
+
+        const aliceTrust = alice.checkUserTrust("@alice:example.com");
+        expect(aliceTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(aliceTrust.isTofu()).toBeTruthy();
+        expect(aliceTrust.isVerified()).toBeTruthy();
+
+        const aliceDeviceTrust = alice.checkDeviceTrust("@alice:example.com", "Osborne2");
+        expect(aliceDeviceTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(aliceDeviceTrust.isLocallyVerified()).toBeTruthy();
+        expect(aliceDeviceTrust.isTofu()).toBeTruthy();
+        expect(aliceDeviceTrust.isVerified()).toBeTruthy();
+    });
+
+    it("should use trust chain to determine device verification", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        // set Alice's cross-signing key
+        await resetCrossSigningKeys(alice);
+        // Alice downloads Bob's ssk and device key
+        const bobMasterSigning = new global.Olm.PkSigning();
+        const bobMasterPrivkey = bobMasterSigning.generate_seed();
+        const bobMasterPubkey = bobMasterSigning.init_with_seed(bobMasterPrivkey);
+        const bobSigning = new global.Olm.PkSigning();
+        const bobPrivkey = bobSigning.generate_seed();
+        const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
+        const bobSSK = {
+            user_id: "@bob:example.com",
+            usage: ["self_signing"],
+            keys: {
+                ["ed25519:" + bobPubkey]: bobPubkey,
+            },
+        };
+        const sskSig = bobMasterSigning.sign(anotherjson.stringify(bobSSK));
+        bobSSK.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobMasterPubkey]: sskSig,
+            },
+        };
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        ["ed25519:" + bobMasterPubkey]: bobMasterPubkey,
+                    },
+                },
+                self_signing: bobSSK,
+            },
+            firstUse: 1,
+            unsigned: {},
+        });
+        const bobDevice = {
+            user_id: "@bob:example.com",
+            device_id: "Dynabook",
+            algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
+            keys: {
+                "curve25519:Dynabook": "somePubkey",
+                "ed25519:Dynabook": "someOtherPubkey",
+            },
+        };
+        const sig = bobSigning.sign(anotherjson.stringify(bobDevice));
+        bobDevice.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobPubkey]: sig,
+            },
+        };
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: bobDevice,
+        });
+        // Bob's device key should be TOFU
+        const bobTrust = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust.isVerified()).toBeFalsy();
+        expect(bobTrust.isTofu()).toBeTruthy();
+
+        const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust.isVerified()).toBeFalsy();
+        expect(bobDeviceTrust.isTofu()).toBeTruthy();
+
+        // Alice verifies Bob's SSK
+        alice.uploadKeySignatures = () => {};
+        await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey, true);
+
+        // Bob's device key should be trusted
+        const bobTrust2 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust2.isCrossSigningVerified()).toBeTruthy();
+        expect(bobTrust2.isTofu()).toBeTruthy();
+
+        const bobDeviceTrust2 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust2.isCrossSigningVerified()).toBeTruthy();
+        expect(bobDeviceTrust2.isLocallyVerified()).toBeFalsy();
+        expect(bobDeviceTrust2.isTofu()).toBeTruthy();
+    });
+
+    it("should trust signatures received from other devices", async function() {
+        const aliceKeys = {};
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            null,
+            aliceKeys,
+        );
+        alice._crypto._deviceList.startTrackingDeviceList("@bob:example.com");
+        alice._crypto._deviceList.stopTrackingAllDeviceLists = () => {};
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+
+        // set Alice's cross-signing key
+        await resetCrossSigningKeys(alice);
+
+        const selfSigningKey = new Uint8Array([
+            0x1e, 0xf4, 0x01, 0x6d, 0x4f, 0xa1, 0x73, 0x66,
+            0x6b, 0xf8, 0x93, 0xf5, 0xb0, 0x4d, 0x17, 0xc0,
+            0x17, 0xb5, 0xa5, 0xf6, 0x59, 0x11, 0x8b, 0x49,
+            0x34, 0xf2, 0x4b, 0x64, 0x9b, 0x52, 0xf8, 0x5f,
+        ]);
+
+        const keyChangePromise = new Promise((resolve, reject) => {
+            alice._crypto._deviceList.once("userCrossSigningUpdated", (userId) => {
+                if (userId === "@bob:example.com") {
+                    resolve();
+                }
+            });
+        });
+
+        const deviceInfo = alice._crypto._deviceList._devices["@alice:example.com"]
+            .Osborne2;
+        const aliceDevice = {
+            user_id: "@alice:example.com",
+            device_id: "Osborne2",
+        };
+        aliceDevice.keys = deviceInfo.keys;
+        aliceDevice.algorithms = deviceInfo.algorithms;
+        await alice._crypto._signObject(aliceDevice);
+
+        const bobOlmAccount = new global.Olm.Account();
+        bobOlmAccount.create();
+        const bobKeys = JSON.parse(bobOlmAccount.identity_keys());
+        const bobDevice = {
+            user_id: "@bob:example.com",
+            device_id: "Dynabook",
+            algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+            keys: {
+                "ed25519:Dynabook": bobKeys.ed25519,
+                "curve25519:Dynabook": bobKeys.curve25519,
+            },
+        };
+        const deviceStr = anotherjson.stringify(bobDevice);
+        bobDevice.signatures = {
+            "@bob:example.com": {
+                "ed25519:Dynabook": bobOlmAccount.sign(deviceStr),
+            },
+        };
+        olmlib.pkSign(bobDevice, selfSigningKey, "@bob:example.com");
+
+        const bobMaster = {
+            user_id: "@bob:example.com",
+            usage: ["master"],
+            keys: {
+                "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk":
+                "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk",
+            },
+        };
+        olmlib.pkSign(bobMaster, aliceKeys.user_signing, "@alice:example.com");
+
+        // Alice downloads Bob's keys
+        // - device key
+        // - ssk
+        // - master key signed by her usk (pretend that it was signed by another
+        //   of Alice's devices)
+        const responses = [
+            HttpResponse.PUSH_RULES_RESPONSE,
+            {
+                method: "POST",
+                path: "/keys/upload",
+                data: {
+                    one_time_key_counts: {
+                        curve25519: 100,
+                        signed_curve25519: 100,
+                    },
+                },
+            },
+            HttpResponse.filterResponse("@alice:example.com"),
+            {
+                method: "GET",
+                path: "/sync",
+                data: {
+                    next_batch: "abcdefg",
+                    device_lists: {
+                        changed: [
+                            "@bob:example.com",
+                        ],
+                    },
+                },
+            },
+            {
+                method: "POST",
+                path: "/keys/query",
+                data: {
+                    "failures": {},
+                    "device_keys": {
+                        "@alice:example.com": {
+                            "Osborne2": aliceDevice,
+                        },
+                        "@bob:example.com": {
+                            "Dynabook": bobDevice,
+                        },
+                    },
+                    "master_keys": {
+                        "@bob:example.com": bobMaster,
+                    },
+                    "self_signing_keys": {
+                        "@bob:example.com": {
+                            user_id: "@bob:example.com",
+                            usage: ["self-signing"],
+                            keys: {
+                                "ed25519:EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ":
+                                "EmkqvokUn8p+vQAGZitOk4PWjp7Ukp3txV2TbMPEiBQ",
+                            },
+                            signatures: {
+                                "@bob:example.com": {
+                                    "ed25519:nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk":
+                                    "2KLiufImvEbfJuAFvsaZD+PsL8ELWl7N1u9yr/9hZvwRghBfQMB"
+                                    + "LAI86b1kDV9+Cq1lt85ykReeCEzmTEPY2BQ",
+                                },
+                            },
+                        },
+                    },
+                },
+            },
+            {
+                method: "POST",
+                path: "/keys/upload",
+                data: {
+                    one_time_key_counts: {
+                        curve25519: 100,
+                        signed_curve25519: 100,
+                    },
+                },
+            },
+        ];
+        setHttpResponses(alice, responses);
+
+        await alice.startClient();
+
+        await keyChangePromise;
+
+        // Bob's device key should be trusted
+        const bobTrust = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(bobTrust.isTofu()).toBeTruthy();
+
+        const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(bobDeviceTrust.isLocallyVerified()).toBeFalsy();
+        expect(bobDeviceTrust.isTofu()).toBeTruthy();
+    });
+
+    it("should dis-trust an unsigned device", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        // set Alice's cross-signing key
+        await resetCrossSigningKeys(alice);
+        // Alice downloads Bob's ssk and device key
+        // (NOTE: device key is not signed by ssk)
+        const bobMasterSigning = new global.Olm.PkSigning();
+        const bobMasterPrivkey = bobMasterSigning.generate_seed();
+        const bobMasterPubkey = bobMasterSigning.init_with_seed(bobMasterPrivkey);
+        const bobSigning = new global.Olm.PkSigning();
+        const bobPrivkey = bobSigning.generate_seed();
+        const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
+        const bobSSK = {
+            user_id: "@bob:example.com",
+            usage: ["self_signing"],
+            keys: {
+                ["ed25519:" + bobPubkey]: bobPubkey,
+            },
+        };
+        const sskSig = bobMasterSigning.sign(anotherjson.stringify(bobSSK));
+        bobSSK.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobMasterPubkey]: sskSig,
+            },
+        };
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        ["ed25519:" + bobMasterPubkey]: bobMasterPubkey,
+                    },
+                },
+                self_signing: bobSSK,
+            },
+            firstUse: 1,
+            unsigned: {},
+        });
+        const bobDevice = {
+            user_id: "@bob:example.com",
+            device_id: "Dynabook",
+            algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
+            keys: {
+                "curve25519:Dynabook": "somePubkey",
+                "ed25519:Dynabook": "someOtherPubkey",
+            },
+        };
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: bobDevice,
+        });
+        // Bob's device key should be untrusted
+        const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust.isVerified()).toBeFalsy();
+        expect(bobDeviceTrust.isTofu()).toBeFalsy();
+
+        // Alice verifies Bob's SSK
+        await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey, true);
+
+        // Bob's device key should be untrusted
+        const bobDeviceTrust2 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust2.isVerified()).toBeFalsy();
+        expect(bobDeviceTrust2.isTofu()).toBeFalsy();
+    });
+
+    it("should dis-trust a user when their ssk changes", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        await resetCrossSigningKeys(alice);
+        // Alice downloads Bob's keys
+        const bobMasterSigning = new global.Olm.PkSigning();
+        const bobMasterPrivkey = bobMasterSigning.generate_seed();
+        const bobMasterPubkey = bobMasterSigning.init_with_seed(bobMasterPrivkey);
+        const bobSigning = new global.Olm.PkSigning();
+        const bobPrivkey = bobSigning.generate_seed();
+        const bobPubkey = bobSigning.init_with_seed(bobPrivkey);
+        const bobSSK = {
+            user_id: "@bob:example.com",
+            usage: ["self_signing"],
+            keys: {
+                ["ed25519:" + bobPubkey]: bobPubkey,
+            },
+        };
+        const sskSig = bobMasterSigning.sign(anotherjson.stringify(bobSSK));
+        bobSSK.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobMasterPubkey]: sskSig,
+            },
+        };
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        ["ed25519:" + bobMasterPubkey]: bobMasterPubkey,
+                    },
+                },
+                self_signing: bobSSK,
+            },
+            firstUse: 1,
+            unsigned: {},
+        });
+        const bobDevice = {
+            user_id: "@bob:example.com",
+            device_id: "Dynabook",
+            algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
+            keys: {
+                "curve25519:Dynabook": "somePubkey",
+                "ed25519:Dynabook": "someOtherPubkey",
+            },
+        };
+        const bobDeviceString = anotherjson.stringify(bobDevice);
+        const sig = bobSigning.sign(bobDeviceString);
+        bobDevice.signatures = {};
+        bobDevice.signatures["@bob:example.com"] = {};
+        bobDevice.signatures["@bob:example.com"]["ed25519:" + bobPubkey] = sig;
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: bobDevice,
+        });
+        // Alice verifies Bob's SSK
+        alice.uploadKeySignatures = () => {};
+        await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey, true);
+
+        // Bob's device key should be trusted
+        const bobDeviceTrust = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust.isVerified()).toBeTruthy();
+        expect(bobDeviceTrust.isTofu()).toBeTruthy();
+
+        // Alice downloads new SSK for Bob
+        const bobMasterSigning2 = new global.Olm.PkSigning();
+        const bobMasterPrivkey2 = bobMasterSigning2.generate_seed();
+        const bobMasterPubkey2 = bobMasterSigning2.init_with_seed(bobMasterPrivkey2);
+        const bobSigning2 = new global.Olm.PkSigning();
+        const bobPrivkey2 = bobSigning2.generate_seed();
+        const bobPubkey2 = bobSigning2.init_with_seed(bobPrivkey2);
+        const bobSSK2 = {
+            user_id: "@bob:example.com",
+            usage: ["self_signing"],
+            keys: {
+                ["ed25519:" + bobPubkey2]: bobPubkey2,
+            },
+        };
+        const sskSig2 = bobMasterSigning2.sign(anotherjson.stringify(bobSSK2));
+        bobSSK2.signatures = {
+            "@bob:example.com": {
+                ["ed25519:" + bobMasterPubkey2]: sskSig2,
+            },
+        };
+        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
+            keys: {
+                master: {
+                    user_id: "@bob:example.com",
+                    usage: ["master"],
+                    keys: {
+                        ["ed25519:" + bobMasterPubkey2]: bobMasterPubkey2,
+                    },
+                },
+                self_signing: bobSSK2,
+            },
+            firstUse: 0,
+            unsigned: {},
+        });
+        // Bob's and his device should be untrusted
+        const bobTrust = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust.isVerified()).toBeFalsy();
+        expect(bobTrust.isTofu()).toBeFalsy();
+
+        const bobDeviceTrust2 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust2.isVerified()).toBeFalsy();
+        expect(bobDeviceTrust2.isTofu()).toBeFalsy();
+
+        // Alice verifies Bob's SSK
+        alice.uploadKeySignatures = () => {};
+        await alice.setDeviceVerified("@bob:example.com", bobMasterPubkey2, true);
+
+        // Bob should be trusted but not his device
+        const bobTrust2 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust2.isVerified()).toBeTruthy();
+
+        const bobDeviceTrust3 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust3.isVerified()).toBeFalsy();
+
+        // Alice gets new signature for device
+        const sig2 = bobSigning2.sign(bobDeviceString);
+        bobDevice.signatures["@bob:example.com"]["ed25519:" + bobPubkey2] = sig2;
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: bobDevice,
+        });
+
+        // Bob's device should be trusted again (but not TOFU)
+        const bobTrust3 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust3.isVerified()).toBeTruthy();
+
+        const bobDeviceTrust4 = alice.checkDeviceTrust("@bob:example.com", "Dynabook");
+        expect(bobDeviceTrust4.isCrossSigningVerified()).toBeTruthy();
+    });
+
+    it("should offer to upgrade device verifications to cross-signing", async function() {
+        let upgradeResolveFunc;
+
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            {
+                cryptoCallbacks: {
+                    shouldUpgradeDeviceVerifications: (verifs) => {
+                        expect(verifs.users["@bob:example.com"]).toBeDefined();
+                        upgradeResolveFunc();
+                        return ["@bob:example.com"];
+                    },
+                },
+            },
+        );
+        const bob = await makeTestClient(
+            {userId: "@bob:example.com", deviceId: "Dynabook"},
+        );
+
+        bob.uploadDeviceSigningKeys = async () => {};
+        bob.uploadKeySignatures = async () => {};
+        // set Bob's cross-signing key
+        await resetCrossSigningKeys(bob);
+        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
+            Dynabook: {
+                algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
+                keys: {
+                    "curve25519:Dynabook": bob._crypto._olmDevice.deviceCurve25519Key,
+                    "ed25519:Dynabook": bob._crypto._olmDevice.deviceEd25519Key,
+                },
+                verified: 1,
+                known: true,
+            },
+        });
+        alice._crypto._deviceList.storeCrossSigningForUser(
+            "@bob:example.com",
+            bob._crypto._crossSigningInfo.toStorage(),
+        );
+
+        alice.uploadDeviceSigningKeys = async () => {};
+        alice.uploadKeySignatures = async () => {};
+        // when alice sets up cross-signing, she should notice that bob's
+        // cross-signing key is signed by his Dynabook, which alice has
+        // verified, and ask if the device verification should be upgraded to a
+        // cross-signing verification
+        let upgradePromise = new Promise((resolve) => {
+            upgradeResolveFunc = resolve;
+        });
+        await resetCrossSigningKeys(alice);
+        await upgradePromise;
+
+        const bobTrust = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust.isCrossSigningVerified()).toBeTruthy();
+        expect(bobTrust.isTofu()).toBeTruthy();
+
+        // "forget" that Bob is trusted
+        delete alice._crypto._deviceList._crossSigningInfo["@bob:example.com"]
+            .keys.master.signatures["@alice:example.com"];
+
+        const bobTrust2 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust2.isCrossSigningVerified()).toBeFalsy();
+        expect(bobTrust2.isTofu()).toBeTruthy();
+
+        upgradePromise = new Promise((resolve) => {
+            upgradeResolveFunc = resolve;
+        });
+        alice._crypto._deviceList.emit("userCrossSigningUpdated", "@bob:example.com");
+        await new Promise((resolve) => {
+            alice._crypto.on("userTrustStatusChanged", resolve);
+        });
+        await upgradePromise;
+
+        const bobTrust3 = alice.checkUserTrust("@bob:example.com");
+        expect(bobTrust3.isCrossSigningVerified()).toBeTruthy();
+        expect(bobTrust3.isTofu()).toBeTruthy();
+    });
+});

spec/unit/crypto/crypto-utils.js

@@ -0,0 +1,44 @@
+import {IndexedDBCryptoStore} from '../../../src/crypto/store/indexeddb-crypto-store';
+
+
+// needs to be phased out and replaced with bootstrapSecretStorage,
+// but that is doing too much extra stuff for it to be an easy transition.
+export async function resetCrossSigningKeys(client, {
+    level,
+    authUploadDeviceSigningKeys = async func => await func(),
+} = {}) {
+    const crypto = client._crypto;
+
+    const oldKeys = Object.assign({}, crypto._crossSigningInfo.keys);
+    try {
+        await crypto._crossSigningInfo.resetKeys(level);
+        await crypto._signObject(crypto._crossSigningInfo.keys.master);
+        // write a copy locally so we know these are trusted keys
+        await crypto._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                crypto._cryptoStore.storeCrossSigningKeys(
+                    txn, crypto._crossSigningInfo.keys);
+            },
+        );
+    } catch (e) {
+        // If anything failed here, revert the keys so we know to try again from the start
+        // next time.
+        crypto._crossSigningInfo.keys = oldKeys;
+        throw e;
+    }
+    crypto._baseApis.emit("crossSigning.keysChanged", {});
+    await crypto._afterCrossSigningLocalKeyChange();
+}
+
+export async function createSecretStorageKey() {
+    const decryption = new global.Olm.PkDecryption();
+    const storagePublicKey = decryption.generate_key();
+    const storagePrivateKey = decryption.get_private_key();
+    decryption.free();
+    return {
+        // `pubkey` not used anymore with symmetric 4S
+        keyInfo: { pubkey: storagePublicKey },
+        privateKey: storagePrivateKey,
+    };
+}

spec/unit/crypto/outgoing-room-key-requests.spec.js

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+import {
+    ROOM_KEY_REQUEST_STATES,
+} from '../../../src/crypto/OutgoingRoomKeyRequestManager';
+
+const requests = [
+    {
+        requestId: "A",
+        requestBody: { session_id: "A", room_id: "A" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "B",
+        requestBody: { session_id: "B", room_id: "B" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "C",
+        requestBody: { session_id: "C", room_id: "C" },
+        state: ROOM_KEY_REQUEST_STATES.UNSENT,
+    },
+];
+
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("Outgoing room key requests [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(async () => {
+        store = dbFactory();
+        await store.startup();
+        await Promise.all(requests.map((request) =>
+            store.getOrAddOutgoingRoomKeyRequest(request),
+        ));
+    });
+
+    it("getAllOutgoingRoomKeyRequestsByState retrieves all entries in a given state",
+    async () => {
+        const r = await
+            store.getAllOutgoingRoomKeyRequestsByState(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(r).toHaveLength(2);
+        requests.filter((e) => e.state == ROOM_KEY_REQUEST_STATES.SENT).forEach((e) => {
+            expect(r).toContainEqual(e);
+        });
+    });
+
+    test("getOutgoingRoomKeyRequestByState retrieves any entry in a given state",
+    async () => {
+        const r =
+            await store.getOutgoingRoomKeyRequestByState([ROOM_KEY_REQUEST_STATES.SENT]);
+        expect(r).not.toBeNull();
+        expect(r).not.toBeUndefined();
+        expect(r.state).toEqual(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(requests).toContainEqual(r);
+    });
+});

spec/unit/crypto/secrets.spec.js

@@ -0,0 +1,674 @@
+/*
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+import * as olmlib from "../../../src/crypto/olmlib";
+import {SECRET_STORAGE_ALGORITHM_V1_AES} from "../../../src/crypto/SecretStorage";
+import {MatrixEvent} from "../../../src/models/event";
+import {TestClient} from '../../TestClient';
+import {makeTestClients} from './verification/util';
+import {encryptAES} from "../../../src/crypto/aes";
+import {resetCrossSigningKeys, createSecretStorageKey} from "./crypto-utils";
+import {logger} from '../../../src/logger';
+
+import * as utils from "../../../src/utils";
+
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    logger.log('nodejs was compiled without crypto support');
+}
+
+async function makeTestClient(userInfo, options) {
+    const client = (new TestClient(
+        userInfo.userId, userInfo.deviceId, undefined, undefined, options,
+    )).client;
+
+    // Make it seem as if we've synced and thus the store can be trusted to
+    // contain valid account data.
+    client.isInitialSyncComplete = function() {
+        return true;
+    };
+
+    await client.initCrypto();
+
+    // No need to download keys for these tests
+    client._crypto.downloadKeys = async function() {};
+
+    return client;
+}
+
+// Wrapper around pkSign to return a signed object. pkSign returns the
+// signature, rather than the signed object.
+function sign(obj, key, userId) {
+    olmlib.pkSign(obj, key, userId);
+    return obj;
+}
+
+describe("Secrets", function() {
+    if (!global.Olm) {
+        logger.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
+    it("should store and retrieve a secret", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+
+        const signing = new global.Olm.PkSigning();
+        const signingKey = signing.generate_seed();
+        const signingPubKey = signing.init_with_seed(signingKey);
+
+        const signingkeyInfo = {
+            user_id: "@alice:example.com",
+            usage: ['master'],
+            keys: {
+                ['ed25519:' + signingPubKey]: signingPubKey,
+            },
+        };
+
+        const getKey = jest.fn(e => {
+            expect(Object.keys(e.keys)).toEqual(["abc"]);
+            return ['abc', key];
+        });
+
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            {
+                cryptoCallbacks: {
+                    getCrossSigningKey: t => signingKey,
+                    getSecretStorageKey: getKey,
+                },
+            },
+        );
+        alice._crypto._crossSigningInfo.setKeys({
+            master: signingkeyInfo,
+        });
+
+        const secretStorage = alice._crypto._secretStorage;
+
+        alice.setAccountData = async function(eventType, contents, callback) {
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                }),
+            ]);
+            if (callback) {
+                callback();
+            }
+        };
+
+        const keyAccountData = {
+            algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
+        };
+        await alice._crypto._crossSigningInfo.signObject(keyAccountData, 'master');
+
+        alice.store.storeAccountDataEvents([
+            new MatrixEvent({
+                type: "m.secret_storage.key.abc",
+                content: keyAccountData,
+            }),
+        ]);
+
+        expect(await secretStorage.isStored("foo")).toBeFalsy();
+
+        await secretStorage.store("foo", "bar", ["abc"]);
+
+        expect(await secretStorage.isStored("foo")).toBeTruthy();
+        expect(await secretStorage.get("foo")).toBe("bar");
+
+        expect(getKey).toHaveBeenCalled();
+    });
+
+    it("should throw if given a key that doesn't exist", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+
+        try {
+            await alice.storeSecret("foo", "bar", ["this secret does not exist"]);
+            // should be able to use expect(...).toThrow() but mocha still fails
+            // the test even when it throws for reasons I have no inclination to debug
+            expect(true).toBeFalsy();
+        } catch (e) {
+        }
+    });
+
+    it("should refuse to encrypt with zero keys", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+
+        try {
+            await alice.storeSecret("foo", "bar", []);
+            expect(true).toBeFalsy();
+        } catch (e) {
+        }
+    });
+
+    it("should encrypt with default key if keys is null", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+        const getKey = jest.fn(e => {
+            expect(Object.keys(e.keys)).toEqual([newKeyId]);
+            return [newKeyId, key];
+        });
+
+        let keys = {};
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+            {
+                cryptoCallbacks: {
+                    getCrossSigningKey: t => keys[t],
+                    saveCrossSigningKeys: k => keys = k,
+                    getSecretStorageKey: getKey,
+                },
+            },
+        );
+        alice.setAccountData = async function(eventType, contents, callback) {
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                }),
+            ]);
+        };
+        resetCrossSigningKeys(alice);
+
+        const { keyId: newKeyId } = await alice.addSecretStorageKey(
+            SECRET_STORAGE_ALGORITHM_V1_AES,
+        );
+        // we don't await on this because it waits for the event to come down the sync
+        // which won't happen in the test setup
+        alice.setDefaultSecretStorageKeyId(newKeyId);
+        await alice.storeSecret("foo", "bar");
+
+        const accountData = alice.getAccountData('foo');
+        expect(accountData.getContent().encrypted).toBeTruthy();
+    });
+
+    it("should refuse to encrypt if no keys given and no default key", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+
+        try {
+            await alice.storeSecret("foo", "bar");
+            expect(true).toBeFalsy();
+        } catch (e) {
+        }
+    });
+
+    it("should request secrets from other clients", async function() {
+        const [osborne2, vax] = await makeTestClients(
+            [
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {userId: "@alice:example.com", deviceId: "VAX"},
+            ],
+            {
+                cryptoCallbacks: {
+                    onSecretRequested: (userId, deviceId, requestId, secretName, deviceTrust) => {
+                        expect(secretName).toBe("foo");
+                        return "bar";
+                    },
+                },
+            },
+        );
+
+        const vaxDevice = vax.client._crypto._olmDevice;
+        const osborne2Device = osborne2.client._crypto._olmDevice;
+        const secretStorage = osborne2.client._crypto._secretStorage;
+
+        osborne2.client._crypto._deviceList.storeDevicesForUser("@alice:example.com", {
+            "VAX": {
+                user_id: "@alice:example.com",
+                device_id: "VAX",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:VAX": vaxDevice.deviceEd25519Key,
+                    "curve25519:VAX": vaxDevice.deviceCurve25519Key,
+                },
+            },
+        });
+        vax.client._crypto._deviceList.storeDevicesForUser("@alice:example.com", {
+            "Osborne2": {
+                user_id: "@alice:example.com",
+                device_id: "Osborne2",
+                algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                keys: {
+                    "ed25519:Osborne2": osborne2Device.deviceEd25519Key,
+                    "curve25519:Osborne2": osborne2Device.deviceCurve25519Key,
+                },
+            },
+        });
+
+        await osborne2Device.generateOneTimeKeys(1);
+        const otks = (await osborne2Device.getOneTimeKeys()).curve25519;
+        await osborne2Device.markKeysAsPublished();
+
+        await vax.client._crypto._olmDevice.createOutboundSession(
+            osborne2Device.deviceCurve25519Key,
+            Object.values(otks)[0],
+        );
+
+        const request = await secretStorage.request("foo", ["VAX"]);
+        const secret = await request.promise;
+
+        expect(secret).toBe("bar");
+    });
+
+    describe("bootstrap", function() {
+        // keys used in some of the tests
+        const XSK = new Uint8Array(
+            olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="),
+        );
+        const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
+        const USK = new Uint8Array(
+            olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="),
+        );
+        const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
+        const SSK = new Uint8Array(
+            olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
+        );
+        const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
+        const SSSSKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
+            ),
+        );
+
+        it("bootstraps when no storage or cross-signing keys locally", async function() {
+            const key = new Uint8Array(16);
+            for (let i = 0; i < 16; i++) key[i] = i;
+            const getKey = jest.fn(e => {
+                return [Object.keys(e.keys)[0], key];
+            });
+
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: getKey,
+                    },
+                },
+            );
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey,
+            });
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("bootstraps when cross-signing keys in secret storage", async function() {
+            const decryption = new global.Olm.PkDecryption();
+            const storagePublicKey = decryption.generate_key();
+            const storagePrivateKey = decryption.get_private_key();
+
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: async request => {
+                            const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
+                            expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
+                            return [defaultKeyId, storagePrivateKey];
+                        },
+                    },
+                },
+            );
+
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+            bob._crypto.checkKeyBackup = async () => {};
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey: async () => ({
+                    // `pubkey` not used anymore with symmetric 4S
+                    keyInfo: { pubkey: storagePublicKey },
+                    privateKey: storagePrivateKey,
+                }),
+            });
+
+            // Clear local cross-signing keys and read from secret storage
+            bob._crypto._deviceList.storeCrossSigningForUser(
+                "@bob:example.com",
+                crossSigning.toStorage(),
+            );
+            crossSigning.keys = {};
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("adds passphrase checking if it's lacking", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                // we never use these values, other than checking that they
+                // exist, so just use dummy values
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "key_id"});
+            const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
+                .getContent();
+            expect(keyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(keyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(keyInfo).toHaveProperty("iv");
+            expect(keyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
+                .toBeTruthy();
+        });
+        it("fixes backup keys in the wrong format", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            key_id: await encryptAES(
+                                "123,45,6,7,89,1,234,56,78,90,12,34,5,67,8,90",
+                                secretStorageKeys.key_id, "m.megolm_backup.v1",
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            const backupKey = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKey.encrypted).toHaveProperty("key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+        });
+    });
+});

spec/unit/crypto/verification/InRoomChannel.spec.js

@@ -0,0 +1,98 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import {InRoomChannel} from "../../../../src/crypto/verification/request/InRoomChannel";
+    "../../../../src/crypto/verification/request/ToDeviceChannel";
+import {MatrixEvent} from "../../../../src/models/event";
+
+describe("InRoomChannel tests", function() {
+    const ALICE = "@alice:hs.tld";
+    const BOB = "@bob:hs.tld";
+    const MALORY = "@malory:hs.tld";
+    const client = {
+        getUserId() { return ALICE; },
+    };
+
+    it("getEventType only returns .request for a message with a msgtype", function() {
+        const invalidEvent = new MatrixEvent({
+            type: "m.key.verification.request",
+        });
+        expect(InRoomChannel.getEventType(invalidEvent)).toStrictEqual("");
+        const validEvent = new MatrixEvent({
+            type: "m.room.message",
+            content: { msgtype: "m.key.verification.request" },
+        });
+        expect(InRoomChannel.getEventType(validEvent)).
+            toStrictEqual("m.key.verification.request");
+        const validFooEvent = new MatrixEvent({ type: "m.foo" });
+        expect(InRoomChannel.getEventType(validFooEvent)).
+            toStrictEqual("m.foo");
+    });
+
+    it("getEventType should return m.room.message for messages", function() {
+        const messageEvent = new MatrixEvent({
+            type: "m.room.message",
+            content: { msgtype: "m.text" },
+        });
+        // XXX: The event type doesn't matter too much, just as long as it's not a verification event
+        expect(InRoomChannel.getEventType(messageEvent)).
+            toStrictEqual("m.room.message");
+    });
+
+    it("getEventType should return actual type for non-message events", function() {
+        const event = new MatrixEvent({
+            type: "m.room.member",
+            content: { },
+        });
+        expect(InRoomChannel.getEventType(event)).
+            toStrictEqual("m.room.member");
+    });
+
+    it("getOtherPartyUserId should not return anything for a request not " +
+        "directed at me", function() {
+        const event = new MatrixEvent({
+            sender: BOB,
+            type: "m.room.message",
+            content: { msgtype: "m.key.verification.request", to: MALORY },
+        });
+        expect(InRoomChannel.getOtherPartyUserId(event, client)).toStrictEqual(undefined);
+    });
+
+    it("getOtherPartyUserId should not return anything an event that is not of a valid " +
+        "request type", function() {
+        // invalid because this should be a room message with msgtype
+        const invalidRequest = new MatrixEvent({
+            sender: BOB,
+            type: "m.key.verification.request",
+            content: { to: ALICE },
+        });
+        expect(InRoomChannel.getOtherPartyUserId(invalidRequest, client))
+            .toStrictEqual(undefined);
+        const startEvent = new MatrixEvent({
+            sender: BOB,
+            type: "m.key.verification.start",
+            content: { to: ALICE },
+        });
+        expect(InRoomChannel.getOtherPartyUserId(startEvent, client))
+            .toStrictEqual(undefined);
+        const fooEvent = new MatrixEvent({
+            sender: BOB,
+            type: "m.foo",
+            content: { to: ALICE },
+        });
+        expect(InRoomChannel.getOtherPartyUserId(fooEvent, client))
+            .toStrictEqual(undefined);
+    });
+});

spec/unit/crypto/verification/qr_code.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018-2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,18 +14,8 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-import logger from '../../../../src/logger';
-
-try {
-    global.Olm = require('olm');
-} catch (e) {
-    logger.warn("unable to run device verification tests: libolm not available");
-}
-
-import expect from 'expect';
-import DeviceInfo from '../../../../lib/crypto/deviceinfo';
-
-import {ShowQRCode, ScanQRCode} from '../../../../lib/crypto/verification/QRCode';
+import "../../../olm-loader";
+import {logger} from "../../../../src/logger";
 
 const Olm = global.Olm;
 
@@ -34,113 +25,17 @@ describe("QR code verification", function() {
         return;
     }
 
-    beforeEach(async function() {
-        await Olm.init();
+    beforeAll(function() {
+        return Olm.init();
     });
 
-    describe("showing", function() {
-        it("should emit an event to show a QR code", async function() {
-            const qrCode = new ShowQRCode({
-                getUserId: () => "@alice:example.com",
-                deviceId: "ABCDEFG",
-                getDeviceEd25519Key: function() {
-                    return "device+ed25519+key";
-                },
-            });
-            const spy = expect.createSpy().andCall((e) => {
-                qrCode.done();
-            });
-            qrCode.on("show_qr_code", spy);
-            await qrCode.verify();
-            expect(spy).toHaveBeenCalledWith({
-                url: "https://matrix.to/#/@alice:example.com?device=ABCDEFG"
-                    + "&action=verify&key_ed25519%3AABCDEFG=device%2Bed25519%2Bkey",
-            });
-        });
-    });
-
-    describe("scanning", function() {
-        const QR_CODE_URL = "https://matrix.to/#/@alice:example.com?device=ABCDEFG"
-              + "&action=verify&key_ed25519%3AABCDEFG=device%2Bed25519%2Bkey";
-        it("should verify when a QR code is sent", async function() {
-            const device = DeviceInfo.fromStorage(
-                {
-                    algorithms: [],
-                    keys: {
-                        "curve25519:ABCDEFG": "device+curve25519+key",
-                        "ed25519:ABCDEFG": "device+ed25519+key",
-                    },
-                    verified: false,
-                    known: false,
-                    unsigned: {},
-                },
-                "ABCDEFG",
-            );
-            const client = {
-                getStoredDevice: expect.createSpy().andReturn(device),
-                setDeviceVerified: expect.createSpy(),
-            };
-            const qrCode = new ScanQRCode(client);
-            qrCode.on("confirm_user_id", ({userId, confirm}) => {
-                if (userId === "@alice:example.com") {
-                    confirm();
-                } else {
-                    qrCode.cancel(new Error("Incorrect user"));
-                }
-            });
-            qrCode.on("scan", ({done}) => {
-                done(QR_CODE_URL);
-            });
-            await qrCode.verify();
-            expect(client.getStoredDevice)
-                .toHaveBeenCalledWith("@alice:example.com", "ABCDEFG");
-            expect(client.setDeviceVerified)
-                .toHaveBeenCalledWith("@alice:example.com", "ABCDEFG");
-        });
-
-        it("should error when the user ID doesn't match", async function() {
-            const client = {
-                getStoredDevice: expect.createSpy(),
-                setDeviceVerified: expect.createSpy(),
-            };
-            const qrCode = new ScanQRCode(client, "@bob:example.com", "ABCDEFG");
-            qrCode.on("scan", ({done}) => {
-                done(QR_CODE_URL);
-            });
-            const spy = expect.createSpy();
-            await qrCode.verify().catch(spy);
-            expect(spy).toHaveBeenCalled();
-            expect(client.getStoredDevice).toNotHaveBeenCalled();
-            expect(client.setDeviceVerified).toNotHaveBeenCalled();
-        });
-
-        it("should error if the key doesn't match", async function() {
-            const device = DeviceInfo.fromStorage(
-                {
-                    algorithms: [],
-                    keys: {
-                        "curve25519:ABCDEFG": "device+curve25519+key",
-                        "ed25519:ABCDEFG": "a+different+device+ed25519+key",
-                    },
-                    verified: false,
-                    known: false,
-                    unsigned: {},
-                },
-                "ABCDEFG",
-            );
-            const client = {
-                getStoredDevice: expect.createSpy().andReturn(device),
-                setDeviceVerified: expect.createSpy(),
-            };
-            const qrCode = new ScanQRCode(client, "@alice:example.com", "ABCDEFG");
-            qrCode.on("scan", ({done}) => {
-                done(QR_CODE_URL);
-            });
-            const spy = expect.createSpy();
-            await qrCode.verify().catch(spy);
-            expect(spy).toHaveBeenCalled();
-            expect(client.getStoredDevice).toHaveBeenCalled();
-            expect(client.setDeviceVerified).toNotHaveBeenCalled();
+    describe("reciprocate", () => {
+        it("should verify the secret", () => {
+            // TODO: Actually write a test for this.
+            // Tests are hard because we are running before the verification
+            // process actually begins, and are largely UI-driven rather than
+            // logic-driven (compared to something like SAS). In the interest
+            // of time, tests are currently excluded.
         });
     });
 });

spec/unit/crypto/verification/request.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,32 +14,29 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-import logger from '../../../../src/logger';
-
-try {
-    global.Olm = require('olm');
-} catch (e) {
-    logger.warn("unable to run device verification tests: libolm not available");
-}
-
-import expect from 'expect';
-
-import {verificationMethods} from '../../../../lib/crypto';
-
-import SAS from '../../../../lib/crypto/verification/SAS';
+import "../../../olm-loader";
+import {verificationMethods} from "../../../../src/crypto";
+import {logger} from "../../../../src/logger";
+import {SAS} from "../../../../src/crypto/verification/SAS";
+import {makeTestClients, setupWebcrypto, teardownWebcrypto} from './util';
 
 const Olm = global.Olm;
 
-import {makeTestClients} from './util';
+jest.useFakeTimers();
 
-describe("verification request", function() {
+describe("verification request integration tests with crypto layer", function() {
     if (!global.Olm) {
         logger.warn('Not running device verification unit tests: libolm not present');
         return;
     }
 
-    beforeEach(async function() {
-        await Olm.init();
+    beforeAll(function() {
+        setupWebcrypto();
+        return Olm.init();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
     });
 
     it("should request and accept a verification", async function() {
@@ -51,7 +49,7 @@ describe("verification request", function() {
                 verificationMethods: [verificationMethods.SAS],
             },
         );
-        alice._crypto._deviceList.getRawStoredDevicesForUser = function() {
+        alice.client._crypto._deviceList.getRawStoredDevicesForUser = function() {
             return {
                 Dynabook: {
                     keys: {
@@ -60,21 +58,23 @@ describe("verification request", function() {
                 },
             };
         };
-        alice.downloadKeys = () => {
+        alice.client.downloadKeys = () => {
             return Promise.resolve();
         };
-        bob.downloadKeys = () => {
+        bob.client.downloadKeys = () => {
             return Promise.resolve();
         };
-        bob.on("crypto.verification.request", (request) => {
+        bob.client.on("crypto.verification.request", (request) => {
             const bobVerifier = request.beginKeyVerification(verificationMethods.SAS);
             bobVerifier.verify();
 
             // XXX: Private function access (but it's a test, so we're okay)
             bobVerifier._endTimer();
         });
-        const aliceVerifier = await alice.requestVerification("@bob:example.com");
-        expect(aliceVerifier).toBeAn(SAS);
+        const aliceRequest = await alice.client.requestVerification("@bob:example.com");
+        await aliceRequest.waitFor(r => r.started);
+        const aliceVerifier = aliceRequest.verifier;
+        expect(aliceVerifier).toBeInstanceOf(SAS);
 
         // XXX: Private function access (but it's a test, so we're okay)
         aliceVerifier._endTimer();

spec/unit/crypto/verification/sas.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018-2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,28 +14,20 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-import logger from '../../../../src/logger';
-
-try {
-    global.Olm = require('olm');
-} catch (e) {
-    logger.warn("unable to run device verification tests: libolm not available");
-}
-
-import expect from 'expect';
-
-import sdk from '../../../..';
-
-import {verificationMethods} from '../../../../lib/crypto';
-import DeviceInfo from '../../../../lib/crypto/deviceinfo';
-
-import SAS from '../../../../lib/crypto/verification/SAS';
+import "../../../olm-loader";
+import {makeTestClients, setupWebcrypto, teardownWebcrypto} from './util';
+import {MatrixEvent} from "../../../../src/models/event";
+import {SAS} from "../../../../src/crypto/verification/SAS";
+import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
+import {verificationMethods} from "../../../../src/crypto";
+import * as olmlib from "../../../../src/crypto/olmlib";
+import {logger} from "../../../../src/logger";
+import {resetCrossSigningKeys} from "../crypto-utils";
 
 const Olm = global.Olm;
 
-const MatrixEvent = sdk.MatrixEvent;
-
-import {makeTestClients} from './util';
+let ALICE_DEVICES;
+let BOB_DEVICES;
 
 describe("SAS verification", function() {
     if (!global.Olm) {
@@ -42,27 +35,40 @@ describe("SAS verification", function() {
         return;
     }
 
-    beforeEach(async function() {
-        await Olm.init();
+    beforeAll(function() {
+        setupWebcrypto();
+        return Olm.init();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
     });
 
     it("should error on an unexpected event", async function() {
-        const sas = new SAS({}, "@alice:example.com", "ABCDEFG");
+        //channel, baseApis, userId, deviceId, startEvent, request
+        const request = {
+            onVerifierCancelled: function() {},
+        };
+        const channel = {
+            send: function() {
+                return Promise.resolve();
+            },
+        };
+        const sas = new SAS(channel, {}, "@alice:example.com", "ABCDEFG", null, request);
         sas.handleEvent(new MatrixEvent({
             sender: "@alice:example.com",
             type: "es.inquisition",
             content: {},
         }));
-        const spy = expect.createSpy();
-        await sas.verify()
-            .catch(spy);
+        const spy = jest.fn();
+        await sas.verify().catch(spy);
         expect(spy).toHaveBeenCalled();
 
         // Cancel the SAS for cleanup (we started a verification, so abort)
         sas.cancel();
     });
 
-    describe("verification", function() {
+    describe("verification", () => {
         let alice;
         let bob;
         let aliceSasEvent;
@@ -70,7 +76,7 @@ describe("SAS verification", function() {
         let aliceVerifier;
         let bobPromise;
 
-        beforeEach(async function() {
+        beforeEach(async () => {
             [alice, bob] = await makeTestClients(
                 [
                     {userId: "@alice:example.com", deviceId: "Osborne2"},
@@ -81,39 +87,44 @@ describe("SAS verification", function() {
                 },
             );
 
-            alice.setDeviceVerified = expect.createSpy();
-            alice.getDeviceEd25519Key = () => {
-                return "alice+base64+ed25519+key";
-            };
-            alice.getStoredDevice = () => {
-                return DeviceInfo.fromStorage(
-                    {
-                        keys: {
-                            "ed25519:Dynabook": "bob+base64+ed25519+key",
-                        },
+            const aliceDevice = alice.client._crypto._olmDevice;
+            const bobDevice = bob.client._crypto._olmDevice;
+
+            ALICE_DEVICES = {
+                Osborne2: {
+                    user_id: "@alice:example.com",
+                    device_id: "Osborne2",
+                    algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                    keys: {
+                        "ed25519:Osborne2": aliceDevice.deviceEd25519Key,
+                        "curve25519:Osborne2": aliceDevice.deviceCurve25519Key,
                     },
-                    "Dynabook",
-                );
+                },
             };
-            alice.downloadKeys = () => {
+
+            BOB_DEVICES = {
+                Dynabook: {
+                    user_id: "@bob:example.com",
+                    device_id: "Dynabook",
+                    algorithms: [olmlib.OLM_ALGORITHM, olmlib.MEGOLM_ALGORITHM],
+                    keys: {
+                        "ed25519:Dynabook": bobDevice.deviceEd25519Key,
+                        "curve25519:Dynabook": bobDevice.deviceCurve25519Key,
+                    },
+                },
+            };
+
+            alice.client._crypto._deviceList.storeDevicesForUser(
+                "@bob:example.com", BOB_DEVICES,
+            );
+            alice.client.downloadKeys = () => {
                 return Promise.resolve();
             };
 
-            bob.setDeviceVerified = expect.createSpy();
-            bob.getStoredDevice = () => {
-                return DeviceInfo.fromStorage(
-                    {
-                        keys: {
-                            "ed25519:Osborne2": "alice+base64+ed25519+key",
-                        },
-                    },
-                    "Osborne2",
-                );
-            };
-            bob.getDeviceEd25519Key = () => {
-                return "bob+base64+ed25519+key";
-            };
-            bob.downloadKeys = () => {
+            bob.client._crypto._deviceList.storeDevicesForUser(
+                "@alice:example.com", ALICE_DEVICES,
+            );
+            bob.client.downloadKeys = () => {
                 return Promise.resolve();
             };
 
@@ -121,8 +132,8 @@ describe("SAS verification", function() {
             bobSasEvent = null;
 
             bobPromise = new Promise((resolve, reject) => {
-                bob.on("crypto.verification.start", (verifier) => {
-                    verifier.on("show_sas", (e) => {
+                bob.client.on("crypto.verification.request", request => {
+                    request.verifier.on("show_sas", (e) => {
                         if (!e.sas.emoji || !e.sas.decimal) {
                             e.cancel();
                         } else if (!aliceSasEvent) {
@@ -138,12 +149,12 @@ describe("SAS verification", function() {
                             }
                         }
                     });
-                    resolve(verifier);
+                    resolve(request.verifier);
                 });
             });
 
-            aliceVerifier = alice.beginKeyVerification(
-                verificationMethods.SAS, bob.getUserId(), bob.deviceId,
+            aliceVerifier = alice.client.beginKeyVerification(
+                verificationMethods.SAS, bob.client.getUserId(), bob.deviceId,
             );
             aliceVerifier.on("show_sas", (e) => {
                 if (!e.sas.emoji || !e.sas.decimal) {
@@ -162,69 +173,163 @@ describe("SAS verification", function() {
                 }
             });
         });
+        afterEach(async () => {
+            await Promise.all([
+                alice.stop(),
+                bob.stop(),
+            ]);
+        });
 
-        it("should verify a key", async function() {
+        it("should verify a key", async () => {
             let macMethod;
-            const origSendToDevice = alice.sendToDevice;
-            bob.sendToDevice = function(type, map) {
+            let keyAgreement;
+            const origSendToDevice = bob.client.sendToDevice.bind(bob.client);
+            bob.client.sendToDevice = function(type, map) {
                 if (type === "m.key.verification.accept") {
-                    macMethod = map[alice.getUserId()][alice.deviceId]
+                    macMethod = map[alice.client.getUserId()][alice.client.deviceId]
                         .message_authentication_code;
+                    keyAgreement = map[alice.client.getUserId()][alice.client.deviceId]
+                        .key_agreement_protocol;
                 }
-                return origSendToDevice.call(this, type, map);
+                return origSendToDevice(type, map);
             };
 
+            alice.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@bob:example.com": BOB_DEVICES,
+                },
+            });
+            bob.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@alice:example.com": ALICE_DEVICES,
+                },
+            });
+
             await Promise.all([
                 aliceVerifier.verify(),
                 bobPromise.then((verifier) => verifier.verify()),
+                alice.httpBackend.flush(),
+                bob.httpBackend.flush(),
             ]);
 
             // make sure that it uses the preferred method
             expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(keyAgreement).toBe("curve25519-hkdf-sha256");
 
             // make sure Alice and Bob verified each other
-            expect(alice.setDeviceVerified)
-                .toHaveBeenCalledWith(bob.getUserId(), bob.deviceId);
-            expect(bob.setDeviceVerified)
-                .toHaveBeenCalledWith(alice.getUserId(), alice.deviceId);
+            const bobDevice
+                  = await alice.client.getStoredDevice("@bob:example.com", "Dynabook");
+            expect(bobDevice.isVerified()).toBeTruthy();
+            const aliceDevice
+                  = await bob.client.getStoredDevice("@alice:example.com", "Osborne2");
+            expect(aliceDevice.isVerified()).toBeTruthy();
         });
 
-        it("should be able to verify using the old MAC", async function() {
+        it("should be able to verify using the old MAC", async () => {
             // pretend that Alice can only understand the old (incorrect) MAC,
             // and make sure that she can still verify with Bob
             let macMethod;
-            const origSendToDevice = alice.sendToDevice;
-            alice.sendToDevice = function(type, map) {
+            const aliceOrigSendToDevice = alice.client.sendToDevice.bind(alice.client);
+            alice.client.sendToDevice = (type, map) => {
                 if (type === "m.key.verification.start") {
                     // Note: this modifies not only the message that Bob
                     // receives, but also the copy of the message that Alice
                     // has, since it is the same object.  If this does not
                     // happen, the verification will fail due to a hash
                     // commitment mismatch.
-                    map[bob.getUserId()][bob.deviceId]
+                    map[bob.client.getUserId()][bob.client.deviceId]
                         .message_authentication_codes = ['hmac-sha256'];
                 }
-                return origSendToDevice.call(this, type, map);
+                return aliceOrigSendToDevice(type, map);
             };
-            bob.sendToDevice = function(type, map) {
+            const bobOrigSendToDevice = bob.client.sendToDevice.bind(bob.client);
+            bob.client.sendToDevice = (type, map) => {
                 if (type === "m.key.verification.accept") {
-                    macMethod = map[alice.getUserId()][alice.deviceId]
+                    macMethod = map[alice.client.getUserId()][alice.client.deviceId]
                         .message_authentication_code;
                 }
-                return origSendToDevice.call(this, type, map);
+                return bobOrigSendToDevice(type, map);
             };
 
+            alice.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@bob:example.com": BOB_DEVICES,
+                },
+            });
+            bob.httpBackend.when('POST', '/keys/query').respond(200, {
+                failures: {},
+                device_keys: {
+                    "@alice:example.com": ALICE_DEVICES,
+                },
+            });
+
             await Promise.all([
                 aliceVerifier.verify(),
                 bobPromise.then((verifier) => verifier.verify()),
+                alice.httpBackend.flush(),
+                bob.httpBackend.flush(),
             ]);
 
             expect(macMethod).toBe("hmac-sha256");
 
-            expect(alice.setDeviceVerified)
-                .toHaveBeenCalledWith(bob.getUserId(), bob.deviceId);
-            expect(bob.setDeviceVerified)
-                .toHaveBeenCalledWith(alice.getUserId(), alice.deviceId);
+            const bobDevice
+                  = await alice.client.getStoredDevice("@bob:example.com", "Dynabook");
+            expect(bobDevice.isVerified()).toBeTruthy();
+            const aliceDevice
+                  = await bob.client.getStoredDevice("@alice:example.com", "Osborne2");
+            expect(aliceDevice.isVerified()).toBeTruthy();
+        });
+
+        it("should verify a cross-signing key", async () => {
+            alice.httpBackend.when('POST', '/keys/device_signing/upload').respond(
+                200, {},
+            );
+            alice.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
+            alice.httpBackend.flush(undefined, 2);
+            await resetCrossSigningKeys(alice.client);
+            bob.httpBackend.when('POST', '/keys/device_signing/upload').respond(200, {});
+            bob.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
+            bob.httpBackend.flush(undefined, 2);
+
+            await resetCrossSigningKeys(bob.client);
+
+            bob.client._crypto._deviceList.storeCrossSigningForUser(
+                "@alice:example.com", {
+                    keys: alice.client._crypto._crossSigningInfo.keys,
+                },
+            );
+
+            const verifyProm = Promise.all([
+                aliceVerifier.verify(),
+                bobPromise.then((verifier) => {
+                    bob.httpBackend.when(
+                        'POST', '/keys/signatures/upload',
+                    ).respond(200, {});
+                    bob.httpBackend.flush(undefined, 1, 2000);
+                    return verifier.verify();
+                }),
+            ]);
+
+            await verifyProm;
+
+            const bobDeviceTrust = alice.client.checkDeviceTrust(
+                "@bob:example.com", "Dynabook",
+            );
+            expect(bobDeviceTrust.isLocallyVerified()).toBeTruthy();
+            expect(bobDeviceTrust.isCrossSigningVerified()).toBeFalsy();
+
+            const aliceTrust = bob.client.checkUserTrust("@alice:example.com");
+            expect(aliceTrust.isCrossSigningVerified()).toBeTruthy();
+            expect(aliceTrust.isTofu()).toBeTruthy();
+
+            const aliceDeviceTrust = bob.client.checkDeviceTrust(
+                "@alice:example.com", "Osborne2",
+            );
+            expect(aliceDeviceTrust.isLocallyVerified()).toBeTruthy();
+            expect(aliceDeviceTrust.isCrossSigningVerified()).toBeFalsy();
         });
     });
 
@@ -238,39 +343,164 @@ describe("SAS verification", function() {
                 verificationMethods: [verificationMethods.SAS],
             },
         );
-        alice.setDeviceVerified = expect.createSpy();
-        alice.downloadKeys = () => {
+        alice.client.setDeviceVerified = jest.fn();
+        alice.client.downloadKeys = () => {
             return Promise.resolve();
         };
-        bob.setDeviceVerified = expect.createSpy();
-        bob.downloadKeys = () => {
+        bob.client.setDeviceVerified = jest.fn();
+        bob.client.downloadKeys = () => {
             return Promise.resolve();
         };
 
         const bobPromise = new Promise((resolve, reject) => {
-            bob.on("crypto.verification.start", (verifier) => {
-                verifier.on("show_sas", (e) => {
+            bob.client.on("crypto.verification.request", request => {
+                request.verifier.on("show_sas", (e) => {
                     e.mismatch();
                 });
-                resolve(verifier);
+                resolve(request.verifier);
             });
         });
 
-        const aliceVerifier = alice.beginKeyVerification(
-            verificationMethods.SAS, bob.getUserId(), bob.deviceId,
+        const aliceVerifier = alice.client.beginKeyVerification(
+            verificationMethods.SAS, bob.client.getUserId(), bob.client.deviceId,
         );
 
-        const aliceSpy = expect.createSpy();
-        const bobSpy = expect.createSpy();
+        const aliceSpy = jest.fn();
+        const bobSpy = jest.fn();
         await Promise.all([
             aliceVerifier.verify().catch(aliceSpy),
             bobPromise.then((verifier) => verifier.verify()).catch(bobSpy),
         ]);
         expect(aliceSpy).toHaveBeenCalled();
         expect(bobSpy).toHaveBeenCalled();
-        expect(alice.setDeviceVerified)
-            .toNotHaveBeenCalled();
-        expect(bob.setDeviceVerified)
-            .toNotHaveBeenCalled();
+        expect(alice.client.setDeviceVerified)
+            .not.toHaveBeenCalled();
+        expect(bob.client.setDeviceVerified)
+            .not.toHaveBeenCalled();
+    });
+
+    describe("verification in DM", function() {
+        let alice;
+        let bob;
+        let aliceSasEvent;
+        let bobSasEvent;
+        let aliceVerifier;
+        let bobPromise;
+
+        beforeEach(async function() {
+            [alice, bob] = await makeTestClients(
+                [
+                    {userId: "@alice:example.com", deviceId: "Osborne2"},
+                    {userId: "@bob:example.com", deviceId: "Dynabook"},
+                ],
+                {
+                    verificationMethods: [verificationMethods.SAS],
+                },
+            );
+
+            alice.client.setDeviceVerified = jest.fn();
+            alice.client.getDeviceEd25519Key = () => {
+                return "alice+base64+ed25519+key";
+            };
+            alice.client.getStoredDevice = () => {
+                return DeviceInfo.fromStorage(
+                    {
+                        keys: {
+                            "ed25519:Dynabook": "bob+base64+ed25519+key",
+                        },
+                    },
+                    "Dynabook",
+                );
+            };
+            alice.client.downloadKeys = () => {
+                return Promise.resolve();
+            };
+
+            bob.client.setDeviceVerified = jest.fn();
+            bob.client.getStoredDevice = () => {
+                return DeviceInfo.fromStorage(
+                    {
+                        keys: {
+                            "ed25519:Osborne2": "alice+base64+ed25519+key",
+                        },
+                    },
+                    "Osborne2",
+                );
+            };
+            bob.client.getDeviceEd25519Key = () => {
+                return "bob+base64+ed25519+key";
+            };
+            bob.client.downloadKeys = () => {
+                return Promise.resolve();
+            };
+
+            aliceSasEvent = null;
+            bobSasEvent = null;
+
+            bobPromise = new Promise((resolve, reject) => {
+                bob.client.on("crypto.verification.request", async (request) => {
+                    const verifier = request.beginKeyVerification(SAS.NAME);
+                    verifier.on("show_sas", (e) => {
+                        if (!e.sas.emoji || !e.sas.decimal) {
+                            e.cancel();
+                        } else if (!aliceSasEvent) {
+                            bobSasEvent = e;
+                        } else {
+                            try {
+                                expect(e.sas).toEqual(aliceSasEvent.sas);
+                                e.confirm();
+                                aliceSasEvent.confirm();
+                            } catch (error) {
+                                e.mismatch();
+                                aliceSasEvent.mismatch();
+                            }
+                        }
+                    });
+                    await verifier.verify();
+                    resolve();
+                });
+            });
+
+            const aliceRequest = await alice.client.requestVerificationDM(
+                bob.client.getUserId(), "!room_id",
+            );
+            await aliceRequest.waitFor(r => r.started);
+            aliceVerifier = aliceRequest.verifier;
+            aliceVerifier.on("show_sas", (e) => {
+                if (!e.sas.emoji || !e.sas.decimal) {
+                    e.cancel();
+                } else if (!bobSasEvent) {
+                    aliceSasEvent = e;
+                } else {
+                    try {
+                        expect(e.sas).toEqual(bobSasEvent.sas);
+                        e.confirm();
+                        bobSasEvent.confirm();
+                    } catch (error) {
+                        e.mismatch();
+                        bobSasEvent.mismatch();
+                    }
+                }
+            });
+        });
+        afterEach(async function() {
+            await Promise.all([
+                alice.stop(),
+                bob.stop(),
+            ]);
+        });
+
+        it("should verify a key", async function() {
+            await Promise.all([
+                aliceVerifier.verify(),
+                bobPromise,
+            ]);
+
+            // make sure Alice and Bob verified each other
+            expect(alice.client.setDeviceVerified)
+                .toHaveBeenCalledWith(bob.client.getUserId(), bob.client.deviceId);
+            expect(bob.client.setDeviceVerified)
+                .toHaveBeenCalledWith(alice.client.getUserId(), alice.client.deviceId);
+        });
     });
 });

spec/unit/crypto/verification/secret_request.spec.js

@@ -0,0 +1,118 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {VerificationBase} from '../../../../src/crypto/verification/Base';
+import {CrossSigningInfo} from '../../../../src/crypto/CrossSigning';
+import {encodeBase64} from "../../../../src/crypto/olmlib";
+import {setupWebcrypto, teardownWebcrypto} from './util';
+
+jest.useFakeTimers();
+
+// Private key for tests only
+const testKey = new Uint8Array([
+    0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+    0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+    0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+    0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+]);
+const testKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
+
+describe("self-verifications", () => {
+    beforeAll(function() {
+        setupWebcrypto();
+        return global.Olm.init();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
+    });
+
+    it("triggers a request for key sharing upon completion", async () => {
+        const userId = "@test:localhost";
+
+        const cacheCallbacks = {
+            getCrossSigningKeyCache: jest.fn().mockReturnValue(null),
+            storeCrossSigningKeyCache: jest.fn(),
+        };
+
+        const _crossSigningInfo = new CrossSigningInfo(
+            userId,
+            {},
+            cacheCallbacks,
+        );
+        _crossSigningInfo.keys = {
+            master: { keys: { X: testKeyPub } },
+            self_signing: { keys: { X: testKeyPub } },
+            user_signing: { keys: { X: testKeyPub } },
+        };
+
+        const _secretStorage = {
+            request: jest.fn().mockReturnValue({
+                promise: Promise.resolve(encodeBase64(testKey)),
+            }),
+        };
+
+        const storeSessionBackupPrivateKey = jest.fn();
+        const restoreKeyBackupWithCache = jest.fn(() => Promise.resolve());
+
+        const client = {
+            _crypto: {
+                _crossSigningInfo,
+                _secretStorage,
+                storeSessionBackupPrivateKey,
+                getSessionBackupPrivateKey: () => null,
+            },
+            requestSecret: _secretStorage.request.bind(_secretStorage),
+            getUserId: () => userId,
+            getKeyBackupVersion: () => Promise.resolve({}),
+            restoreKeyBackupWithCache,
+        };
+
+        const request = {
+            onVerifierFinished: () => undefined,
+        };
+
+        const verification = new VerificationBase(
+            undefined, // channel
+            client, // baseApis
+            userId,
+            "ABC", // deviceId
+            undefined, // startEvent
+            request,
+        );
+        verification._resolve = () => undefined;
+
+        const result = await verification.done();
+
+        /* We should request, and store, 3 cross signing keys and the key backup key */
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(3);
+        expect(_secretStorage.request.mock.calls.length).toBe(4);
+
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
+          .toEqual(testKey);
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[1][1])
+          .toEqual(testKey);
+
+        expect(storeSessionBackupPrivateKey.mock.calls[0][0])
+          .toEqual(testKey);
+
+        expect(restoreKeyBackupWithCache).toHaveBeenCalled();
+
+        expect(result).toBeInstanceOf(Array);
+        expect(result[0][0]).toBe(testKeyPub);
+        expect(result[1][0]).toBe(testKeyPub);
+    });
+});

spec/unit/crypto/verification/util.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,50 +15,104 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import TestClient from '../../../TestClient';
-
-import sdk from '../../../..';
-const MatrixEvent = sdk.MatrixEvent;
+import {TestClient} from '../../../TestClient';
+import {MatrixEvent} from "../../../../src/models/event";
+import nodeCrypto from "crypto";
+import {logger} from '../../../../src/logger';
 
 export async function makeTestClients(userInfos, options) {
     const clients = [];
     const clientMap = {};
     const sendToDevice = function(type, map) {
-        // console.log(this.getUserId(), "sends", type, map);
+        // logger.log(this.getUserId(), "sends", type, map);
         for (const [userId, devMap] of Object.entries(map)) {
             if (userId in clientMap) {
                 for (const [deviceId, msg] of Object.entries(devMap)) {
                     if (deviceId in clientMap[userId]) {
                         const event = new MatrixEvent({
-                            sender: this.getUserId(), // eslint-disable-line babel/no-invalid-this
+                            sender: this.getUserId(), // eslint-disable-line @babel/no-invalid-this
                             type: type,
                             content: msg,
                         });
-                        setTimeout(
-                            () => clientMap[userId][deviceId]
-                                .emit("toDeviceEvent", event),
-                            0,
+                        const client = clientMap[userId][deviceId];
+                        const decryptionPromise = event.isEncrypted() ?
+                            event.attemptDecryption(client._crypto) :
+                            Promise.resolve();
+
+                        decryptionPromise.then(
+                            () => client.emit("toDeviceEvent", event),
                         );
                     }
                 }
             }
         }
     };
+    const sendEvent = function(room, type, content) {
+        // make up a unique ID as the event ID
+        const eventId = "$" + this.makeTxnId(); // eslint-disable-line @babel/no-invalid-this
+        const rawEvent = {
+            sender: this.getUserId(), // eslint-disable-line @babel/no-invalid-this
+            type: type,
+            content: content,
+            room_id: room,
+            event_id: eventId,
+            origin_server_ts: Date.now(),
+        };
+        const event = new MatrixEvent(rawEvent);
+        const remoteEcho = new MatrixEvent(Object.assign({}, rawEvent, {
+            unsigned: {
+                transaction_id: this.makeTxnId(), // eslint-disable-line @babel/no-invalid-this
+            },
+        }));
+
+        setImmediate(() => {
+            for (const tc of clients) {
+                if (tc.client === this) { // eslint-disable-line @babel/no-invalid-this
+                    logger.log("sending remote echo!!");
+                    tc.client.emit("Room.timeline", remoteEcho);
+                } else {
+                    tc.client.emit("Room.timeline", event);
+                }
+            }
+        });
+
+        return Promise.resolve({event_id: eventId});
+    };
 
     for (const userInfo of userInfos) {
-        const client = (new TestClient(
+        let keys = {};
+        if (!options) options = {};
+        if (!options.cryptoCallbacks) options.cryptoCallbacks = {};
+        if (!options.cryptoCallbacks.saveCrossSigningKeys) {
+            options.cryptoCallbacks.saveCrossSigningKeys = k => { keys = k; };
+            options.cryptoCallbacks.getCrossSigningKey = typ => keys[typ];
+        }
+        const testClient = new TestClient(
             userInfo.userId, userInfo.deviceId, undefined, undefined,
             options,
-        )).client;
+        );
         if (!(userInfo.userId in clientMap)) {
             clientMap[userInfo.userId] = {};
         }
-        clientMap[userInfo.userId][userInfo.deviceId] = client;
-        client.sendToDevice = sendToDevice;
-        clients.push(client);
+        clientMap[userInfo.userId][userInfo.deviceId] = testClient.client;
+        testClient.client.sendToDevice = sendToDevice;
+        testClient.client.sendEvent = sendEvent;
+        clients.push(testClient);
     }
 
-    await Promise.all(clients.map((client) => client.initCrypto()));
+    await Promise.all(clients.map((testClient) => testClient.client.initCrypto()));
 
     return clients;
 }
+
+export function setupWebcrypto() {
+    global.crypto = {
+        getRandomValues: (buf) => {
+            return nodeCrypto.randomFillSync(buf);
+        },
+    };
+}
+
+export function teardownWebcrypto() {
+    global.crypto = undefined;
+}

spec/unit/crypto/verification/verification_request.spec.js

@@ -0,0 +1,285 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+import {VerificationRequest, READY_TYPE, START_TYPE, DONE_TYPE} from
+    "../../../../src/crypto/verification/request/VerificationRequest";
+import {InRoomChannel} from "../../../../src/crypto/verification/request/InRoomChannel";
+import {ToDeviceChannel} from
+    "../../../../src/crypto/verification/request/ToDeviceChannel";
+import {MatrixEvent} from "../../../../src/models/event";
+import {setupWebcrypto, teardownWebcrypto} from "./util";
+
+function makeMockClient(userId, deviceId) {
+    let counter = 1;
+    let events = [];
+    const deviceEvents = {};
+    return {
+        getUserId() { return userId; },
+        getDeviceId() { return deviceId; },
+
+        sendEvent(roomId, type, content) {
+            counter = counter + 1;
+            const eventId = `$${userId}-${deviceId}-${counter}`;
+            events.push(new MatrixEvent({
+                sender: userId,
+                event_id: eventId,
+                room_id: roomId,
+                type,
+                content,
+                origin_server_ts: Date.now(),
+            }));
+            return Promise.resolve({event_id: eventId});
+        },
+
+        sendToDevice(type, msgMap) {
+            for (const userId of Object.keys(msgMap)) {
+                const deviceMap = msgMap[userId];
+                for (const deviceId of Object.keys(deviceMap)) {
+                    const content = deviceMap[deviceId];
+                    const event = new MatrixEvent({content, type});
+                    deviceEvents[userId] = deviceEvents[userId] || {};
+                    deviceEvents[userId][deviceId] = deviceEvents[userId][deviceId] || [];
+                    deviceEvents[userId][deviceId].push(event);
+                }
+            }
+            return Promise.resolve();
+        },
+
+        popEvents() {
+            const e = events;
+            events = [];
+            return e;
+        },
+
+        popDeviceEvents(userId, deviceId) {
+            const forDevice = deviceEvents[userId];
+            const events = forDevice && forDevice[deviceId];
+            const result = events || [];
+            if (events) {
+                delete forDevice[deviceId];
+            }
+            return result;
+        },
+    };
+}
+
+const MOCK_METHOD = "mock-verify";
+class MockVerifier {
+    constructor(channel, client, userId, deviceId, startEvent) {
+        this._channel = channel;
+        this._startEvent = startEvent;
+    }
+
+    get events() {
+        return [DONE_TYPE];
+    }
+
+    async start() {
+        if (this._startEvent) {
+            await this._channel.send(DONE_TYPE, {});
+        } else {
+            await this._channel.send(START_TYPE, {method: MOCK_METHOD});
+        }
+    }
+
+    async handleEvent(event) {
+        if (event.getType() === DONE_TYPE && !this._startEvent) {
+            await this._channel.send(DONE_TYPE, {});
+        }
+    }
+
+    canSwitchStartEvent() {
+        return false;
+    }
+}
+
+function makeRemoteEcho(event) {
+    return new MatrixEvent(Object.assign({}, event.event, {
+        unsigned: {
+            transaction_id: "abc",
+        },
+    }));
+}
+
+async function distributeEvent(ownRequest, theirRequest, event) {
+    await ownRequest.channel.handleEvent(
+            makeRemoteEcho(event), ownRequest, true);
+    await theirRequest.channel.handleEvent(event, theirRequest, true);
+}
+
+jest.useFakeTimers();
+
+describe("verification request unit tests", function() {
+    beforeAll(function() {
+        setupWebcrypto();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
+    });
+
+    it("transition from UNSENT to DONE through happy path", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()),
+            new Map([[MOCK_METHOD, MockVerifier]]), alice);
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"),
+            new Map([[MOCK_METHOD, MockVerifier]]), bob);
+        expect(aliceRequest.invalid).toBe(true);
+        expect(bobRequest.invalid).toBe(true);
+
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        expect(requestEvent.getType()).toBe("m.room.message");
+        await distributeEvent(aliceRequest, bobRequest, requestEvent);
+        expect(aliceRequest.requested).toBe(true);
+        expect(bobRequest.requested).toBe(true);
+
+        await bobRequest.accept();
+        const [readyEvent] = bob.popEvents();
+        expect(readyEvent.getType()).toBe(READY_TYPE);
+        await distributeEvent(bobRequest, aliceRequest, readyEvent);
+        expect(bobRequest.ready).toBe(true);
+        expect(aliceRequest.ready).toBe(true);
+
+        const verifier = aliceRequest.beginKeyVerification(MOCK_METHOD);
+        await verifier.start();
+        const [startEvent] = alice.popEvents();
+        expect(startEvent.getType()).toBe(START_TYPE);
+        await distributeEvent(aliceRequest, bobRequest, startEvent);
+        expect(aliceRequest.started).toBe(true);
+        expect(aliceRequest.verifier).toBeInstanceOf(MockVerifier);
+        expect(bobRequest.started).toBe(true);
+        expect(bobRequest.verifier).toBeInstanceOf(MockVerifier);
+
+        await bobRequest.verifier.start();
+        const [bobDoneEvent] = bob.popEvents();
+        expect(bobDoneEvent.getType()).toBe(DONE_TYPE);
+        await distributeEvent(bobRequest, aliceRequest, bobDoneEvent);
+        const [aliceDoneEvent] = alice.popEvents();
+        expect(aliceDoneEvent.getType()).toBe(DONE_TYPE);
+        await distributeEvent(aliceRequest, bobRequest, aliceDoneEvent);
+        expect(aliceRequest.done).toBe(true);
+        expect(bobRequest.done).toBe(true);
+    });
+
+    it("methods only contains common methods", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()),
+            new Map([["c", function() {}], ["a", function() {}]]), alice);
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"),
+            new Map([["c", function() {}], ["b", function() {}]]), bob);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        await distributeEvent(aliceRequest, bobRequest, requestEvent);
+        await bobRequest.accept();
+        const [readyEvent] = bob.popEvents();
+        await distributeEvent(bobRequest, aliceRequest, readyEvent);
+        expect(aliceRequest.methods).toStrictEqual(["c"]);
+        expect(bobRequest.methods).toStrictEqual(["c"]);
+    });
+
+    it("other client accepting request puts it in observeOnly mode", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob1 = makeMockClient("@bob:matrix.tld", "device1");
+        const bob2 = makeMockClient("@bob:matrix.tld", "device2");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob1.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        const bob1Request = new VerificationRequest(
+            new InRoomChannel(bob1, "!room"), new Map(), bob1);
+        const bob2Request = new VerificationRequest(
+            new InRoomChannel(bob2, "!room"), new Map(), bob2);
+
+        await bob1Request.channel.handleEvent(requestEvent, bob1Request, true);
+        await bob2Request.channel.handleEvent(requestEvent, bob2Request, true);
+
+        await bob1Request.accept();
+        const [readyEvent] = bob1.popEvents();
+        expect(bob2Request.observeOnly).toBe(false);
+        await bob2Request.channel.handleEvent(readyEvent, bob2Request, true);
+        expect(bob2Request.observeOnly).toBe(true);
+    });
+
+    it("verify own device with to_device messages", async function() {
+        const bob1 = makeMockClient("@bob:matrix.tld", "device1");
+        const bob2 = makeMockClient("@bob:matrix.tld", "device2");
+        const bob1Request = new VerificationRequest(
+            new ToDeviceChannel(bob1, bob1.getUserId(), ["device1", "device2"],
+                ToDeviceChannel.makeTransactionId(), "device2"),
+            new Map([[MOCK_METHOD, MockVerifier]]), bob1);
+        const to = {userId: "@bob:matrix.tld", deviceId: "device2"};
+        const verifier = bob1Request.beginKeyVerification(MOCK_METHOD, to);
+        expect(verifier).toBeInstanceOf(MockVerifier);
+        await verifier.start();
+        const [startEvent] = bob1.popDeviceEvents(to.userId, to.deviceId);
+        expect(startEvent.getType()).toBe(START_TYPE);
+        const bob2Request = new VerificationRequest(
+            new ToDeviceChannel(bob2, bob2.getUserId(), ["device1"]),
+            new Map([[MOCK_METHOD, MockVerifier]]), bob2);
+
+        await bob2Request.channel.handleEvent(startEvent, bob2Request, true);
+        await bob2Request.verifier.start();
+        const [doneEvent1] = bob2.popDeviceEvents("@bob:matrix.tld", "device1");
+        expect(doneEvent1.getType()).toBe(DONE_TYPE);
+        await bob1Request.channel.handleEvent(doneEvent1, bob1Request, true);
+        const [doneEvent2] = bob1.popDeviceEvents("@bob:matrix.tld", "device2");
+        expect(doneEvent2.getType()).toBe(DONE_TYPE);
+        await bob2Request.channel.handleEvent(doneEvent2, bob2Request, true);
+
+        expect(bob1Request.done).toBe(true);
+        expect(bob2Request.done).toBe(true);
+    });
+
+    it("request times out after 10 minutes", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        await aliceRequest.channel.handleEvent(requestEvent, aliceRequest, true,
+            true, true);
+
+        expect(aliceRequest.cancelled).toBe(false);
+        expect(aliceRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(10 * 60 * 1000);
+        expect(aliceRequest._cancellingUserId).toBe(alice.getUserId());
+    });
+
+    it("request times out 2 minutes after receipt", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"), new Map(), bob);
+
+        await bobRequest.channel.handleEvent(requestEvent, bobRequest, true);
+
+        expect(bobRequest.cancelled).toBe(false);
+        expect(bobRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(2 * 60 * 1000);
+        expect(bobRequest._cancellingUserId).toBe(bob.getUserId());
+    });
+});

spec/unit/event-timeline.spec.js

@@ -1,16 +1,12 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const EventTimeline = sdk.EventTimeline;
-const utils = require("../test-utils");
+import * as utils from "../test-utils";
+import {EventTimeline} from "../../src/models/event-timeline";
+import {RoomState} from "../../src/models/room-state";
 
 function mockRoomStates(timeline) {
-    timeline._startState = utils.mock(sdk.RoomState, "startState");
-    timeline._endState = utils.mock(sdk.RoomState, "endState");
+    timeline._startState = utils.mock(RoomState, "startState");
+    timeline._endState = utils.mock(RoomState, "endState");
 }
 
-import expect from 'expect';
-
 describe("EventTimeline", function() {
     const roomId = "!foo:bar";
     const userA = "@alice:bar";
@@ -18,8 +14,6 @@ describe("EventTimeline", function() {
     let timeline;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-
         // XXX: this is a horrid hack; should use sinon or something instead to mock
         const timelineSet = { room: { roomId: roomId }};
         timelineSet.room.getUnfilteredTimelineSet = function() {
@@ -78,7 +72,7 @@ describe("EventTimeline", function() {
 
             expect(function() {
                 timeline.initialiseState(state);
-            }).toNotThrow();
+            }).not.toThrow();
             timeline.addEvent(event, false);
             expect(function() {
                 timeline.initialiseState(state);
@@ -121,7 +115,7 @@ describe("EventTimeline", function() {
             const next = {b: "b"};
             expect(function() {
                 timeline.setNeighbouringTimeline(prev, EventTimeline.BACKWARDS);
-            }).toNotThrow();
+            }).not.toThrow();
             expect(timeline.getNeighbouringTimeline(EventTimeline.BACKWARDS))
                 .toBe(prev);
             expect(function() {
@@ -130,7 +124,7 @@ describe("EventTimeline", function() {
 
             expect(function() {
                 timeline.setNeighbouringTimeline(next, EventTimeline.FORWARDS);
-            }).toNotThrow();
+            }).not.toThrow();
             expect(timeline.getNeighbouringTimeline(EventTimeline.FORWARDS))
                 .toBe(next);
             expect(function() {
@@ -187,14 +181,14 @@ describe("EventTimeline", function() {
                 name: "Old Alice",
             };
             timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCall(function(uid) {
+                .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
             timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCall(function(uid) {
+                .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
                     }
@@ -229,14 +223,14 @@ describe("EventTimeline", function() {
                 name: "Old Alice",
             };
             timeline.getState(EventTimeline.FORWARDS).getSentinelMember
-                .andCall(function(uid) {
+                .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return sentinel;
                     }
                     return null;
                 });
             timeline.getState(EventTimeline.BACKWARDS).getSentinelMember
-                .andCall(function(uid) {
+                .mockImplementation(function(uid) {
                     if (uid === userA) {
                         return oldSentinel;
                     }
@@ -281,7 +275,7 @@ describe("EventTimeline", function() {
             expect(events[1].forwardLooking).toBe(true);
 
             expect(timeline.getState(EventTimeline.BACKWARDS).setStateEvents).
-                toNotHaveBeenCalled();
+                not.toHaveBeenCalled();
         });
 
 
@@ -311,7 +305,7 @@ describe("EventTimeline", function() {
             expect(events[1].forwardLooking).toBe(false);
 
             expect(timeline.getState(EventTimeline.FORWARDS).setStateEvents).
-                toNotHaveBeenCalled();
+                not.toHaveBeenCalled();
         });
     });
 

spec/unit/event.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 New Vector Ltd
+Copyright 2019 The Matrix.org Foundaction C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,20 +15,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import sdk from '../..';
-const MatrixEvent = sdk.MatrixEvent;
-
-import testUtils from '../test-utils';
-
-import expect from 'expect';
-import Promise from 'bluebird';
-import logger from '../../src/logger';
+import {logger} from "../../src/logger";
+import {MatrixEvent} from "../../src/models/event";
 
 describe("MatrixEvent", () => {
-    beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-    });
-
     describe(".attemptDecryption", () => {
         let encryptedEvent;
 
@@ -45,6 +36,7 @@ describe("MatrixEvent", () => {
             let callCount = 0;
 
             let prom2;
+            let prom2Fulfilled = false;
 
             const crypto = {
                 decryptEvent: function() {
@@ -54,12 +46,13 @@ describe("MatrixEvent", () => {
                         // schedule a second decryption attempt while
                         // the first one is still running.
                         prom2 = encryptedEvent.attemptDecryption(crypto);
+                        prom2.then(() => prom2Fulfilled = true);
 
                         const error = new Error("nope");
                         error.name = 'DecryptionError';
                         return Promise.reject(error);
                     } else {
-                        expect(prom2.isFulfilled()).toBe(
+                        expect(prom2Fulfilled).toBe(
                             false, 'second attemptDecryption resolved too soon');
 
                         return Promise.resolve({

spec/unit/filter-component.spec.js

@@ -0,0 +1,34 @@
+import {FilterComponent} from "../../src/filter-component";
+import {mkEvent} from '../test-utils';
+
+describe("Filter Component", function() {
+    describe("types", function() {
+        it("should filter out events with other types", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.member',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(false);
+        });
+
+        it("should validate events with the same type", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.message',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(true);
+        });
+    });
+});

spec/unit/filter.spec.js

@@ -1,10 +1,4 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const Filter = sdk.Filter;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import {Filter} from "../../src/filter";
 
 describe("Filter", function() {
     const filterId = "f1lt3ring15g00d4ursoul";
@@ -12,7 +6,6 @@ describe("Filter", function() {
     let filter;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         filter = new Filter(userId);
     });
 

spec/unit/interactive-auth.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,18 +14,10 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const utils = require("../test-utils");
-
-const InteractiveAuth = sdk.InteractiveAuth;
-const MatrixError = sdk.MatrixError;
-
-import expect from 'expect';
-import logger from '../../src/logger';
+import {logger} from "../../src/logger";
+import {InteractiveAuth} from "../../src/interactive-auth";
+import {MatrixError} from "../../src/http-api";
 
 // Trivial client object to test interactive auth
 // (we do not need TestClient here)
@@ -35,13 +28,9 @@ class FakeClient {
 }
 
 describe("InteractiveAuth", function() {
-    beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-    });
-
-    it("should start an auth stage and complete it", function(done) {
-        const doRequest = expect.createSpy();
-        const stateUpdated = expect.createSpy();
+    it("should start an auth stage and complete it", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
 
         const ia = new InteractiveAuth({
             matrixClient: new FakeClient(),
@@ -64,7 +53,7 @@ describe("InteractiveAuth", function() {
         });
 
         // first we expect a call here
-        stateUpdated.andCall(function(stage) {
+        stateUpdated.mockImplementation(function(stage) {
             logger.log('aaaa');
             expect(stage).toEqual("logintype");
             ia.submitAuthDict({
@@ -75,7 +64,7 @@ describe("InteractiveAuth", function() {
 
         // .. which should trigger a call here
         const requestRes = {"a": "b"};
-        doRequest.andCall(function(authData) {
+        doRequest.mockImplementation(function(authData) {
             logger.log('cccc');
             expect(authData).toEqual({
                 session: "sessionId",
@@ -85,16 +74,16 @@ describe("InteractiveAuth", function() {
             return Promise.resolve(requestRes);
         });
 
-        ia.attemptAuth().then(function(res) {
+        return ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
-            expect(doRequest.calls.length).toEqual(1);
-            expect(stateUpdated.calls.length).toEqual(1);
-        }).nodeify(done);
+            expect(doRequest).toBeCalledTimes(1);
+            expect(stateUpdated).toBeCalledTimes(1);
+        });
     });
 
-    it("should make a request if no authdata is provided", function(done) {
-        const doRequest = expect.createSpy();
-        const stateUpdated = expect.createSpy();
+    it("should make a request if no authdata is provided", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
 
         const ia = new InteractiveAuth({
             matrixClient: new FakeClient(),
@@ -106,9 +95,9 @@ describe("InteractiveAuth", function() {
         expect(ia.getStageParams("logintype")).toBe(undefined);
 
         // first we expect a call to doRequest
-        doRequest.andCall(function(authData) {
+        doRequest.mockImplementation(function(authData) {
             logger.log("request1", authData);
-            expect(authData).toEqual({});
+            expect(authData).toEqual(null); // first request should be null
             const err = new MatrixError({
                 session: "sessionId",
                 flows: [
@@ -124,7 +113,7 @@ describe("InteractiveAuth", function() {
 
         // .. which should be followed by a call to stateUpdated
         const requestRes = {"a": "b"};
-        stateUpdated.andCall(function(stage) {
+        stateUpdated.mockImplementation(function(stage) {
             expect(stage).toEqual("logintype");
             expect(ia.getSessionId()).toEqual("sessionId");
             expect(ia.getStageParams("logintype")).toEqual({
@@ -132,7 +121,7 @@ describe("InteractiveAuth", function() {
             });
 
             // submitAuthDict should trigger another call to doRequest
-            doRequest.andCall(function(authData) {
+            doRequest.mockImplementation(function(authData) {
                 logger.log("request2", authData);
                 expect(authData).toEqual({
                     session: "sessionId",
@@ -148,10 +137,39 @@ describe("InteractiveAuth", function() {
             });
         });
 
-        ia.attemptAuth().then(function(res) {
+        return ia.attemptAuth().then(function(res) {
             expect(res).toBe(requestRes);
-            expect(doRequest.calls.length).toEqual(2);
-            expect(stateUpdated.calls.length).toEqual(1);
-        }).nodeify(done);
+            expect(doRequest).toBeCalledTimes(2);
+            expect(stateUpdated).toBeCalledTimes(1);
+        });
+    });
+
+    it("should start an auth stage and reject if no auth flow", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+        });
+
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual(null); // first request should be null
+            const err = new MatrixError({
+                session: "sessionId",
+                flows: [],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        return ia.attemptAuth().catch(function(error) {
+            expect(error.message).toBe('No appropriate authentication flow found');
+        });
     });
 });

spec/unit/login.spec.js

@@ -1,5 +1,4 @@
-import expect from 'expect';
-import TestClient from '../TestClient';
+import {TestClient} from '../TestClient';
 
 describe('Login request', function() {
     let client;

spec/unit/matrix-client.spec.js

@@ -1,13 +1,8 @@
-"use strict";
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const MatrixClient = sdk.MatrixClient;
-const utils = require("../test-utils");
+import {logger} from "../../src/logger";
+import {MatrixClient} from "../../src/client";
+import {Filter} from "../../src/filter";
 
-import expect from 'expect';
-import lolex from 'lolex';
-import logger from '../../src/logger';
+jest.useFakeTimers();
 
 describe("MatrixClient", function() {
     const userId = "@alice:bar";
@@ -16,7 +11,6 @@ describe("MatrixClient", function() {
     let client;
     let store;
     let scheduler;
-    let clock;
 
     const KEEP_ALIVE_PATH = "/_matrix/client/versions";
 
@@ -85,7 +79,7 @@ describe("MatrixClient", function() {
                 );
             }
             pendingLookup = {
-                promise: Promise.defer().promise,
+                promise: new Promise(() => {}),
                 method: method,
                 path: path,
             };
@@ -121,28 +115,26 @@ describe("MatrixClient", function() {
             return Promise.resolve(next.data);
         }
         expect(true).toBe(false, "Expected different request. " + logLine);
-        return Promise.defer().promise;
+        return new Promise(() => {});
     }
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        clock = lolex.install();
         scheduler = [
             "getQueueForEvent", "queueEvent", "removeEventFromQueue",
             "setProcessFunction",
-        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
+        ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
         store = [
             "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback",
             "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom", "storeUser",
             "getFilterIdByName", "setFilterIdByName", "getFilter", "storeFilter",
             "getSyncAccumulator", "startup", "deleteAllData",
-        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
-        store.getSavedSync = expect.createSpy().andReturn(Promise.resolve(null));
-        store.getSavedSyncToken = expect.createSpy().andReturn(Promise.resolve(null));
-        store.setSyncData = expect.createSpy().andReturn(Promise.resolve(null));
-        store.getClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
-        store.storeClientOptions = expect.createSpy().andReturn(Promise.resolve(null));
-        store.isNewlyCreated = expect.createSpy().andReturn(Promise.resolve(true));
+        ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
+        store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.getClientOptions = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.storeClientOptions = jest.fn().mockReturnValue(Promise.resolve(null));
+        store.isNewlyCreated = jest.fn().mockReturnValue(Promise.resolve(true));
         client = new MatrixClient({
             baseUrl: "https://my.home.server",
             idBaseUrl: identityServerUrl,
@@ -154,13 +146,10 @@ describe("MatrixClient", function() {
         });
         // FIXME: We shouldn't be yanking _http like this.
         client._http = [
-            "authedRequest", "authedRequestWithPrefix", "getContentUri",
-            "request", "requestWithPrefix", "uploadContent",
-        ].reduce((r, k) => { r[k] = expect.createSpy(); return r; }, {});
-        client._http.authedRequest.andCall(httpReq);
-        client._http.authedRequestWithPrefix.andCall(httpReq);
-        client._http.requestWithPrefix.andCall(httpReq);
-        client._http.request.andCall(httpReq);
+            "authedRequest", "getContentUri", "request", "uploadContent",
+        ].reduce((r, k) => { r[k] = jest.fn(); return r; }, {});
+        client._http.authedRequest.mockImplementation(httpReq);
+        client._http.request.mockImplementation(httpReq);
 
         // set reasonable working defaults
         acceptKeepalives = true;
@@ -172,17 +161,13 @@ describe("MatrixClient", function() {
     });
 
     afterEach(function() {
-        clock.uninstall();
         // need to re-stub the requests with NOPs because there are no guarantees
         // clients from previous tests will be GC'd before the next test. This
         // means they may call /events and then fail an expect() which will fail
         // a DIFFERENT test (pollution between tests!) - we return unresolved
         // promises to stop the client from continuing to run.
-        client._http.authedRequest.andCall(function() {
-            return Promise.defer().promise;
-        });
-        client._http.authedRequestWithPrefix.andCall(function() {
-            return Promise.defer().promise;
+        client._http.authedRequest.mockImplementation(function() {
+            return new Promise(() => {});
         });
     });
 
@@ -191,10 +176,10 @@ describe("MatrixClient", function() {
         httpLookups.push(PUSH_RULES_RESPONSE);
         httpLookups.push(SYNC_RESPONSE);
         const filterId = "ehfewf";
-        store.getFilterIdByName.andReturn(filterId);
-        const filter = new sdk.Filter(0, filterId);
+        store.getFilterIdByName.mockReturnValue(filterId);
+        const filter = new Filter(0, filterId);
         filter.setDefinition({"room": {"timeline": {"limit": 8}}});
-        store.getFilter.andReturn(filter);
+        store.getFilter.mockReturnValue(filter);
         const syncPromise = new Promise((resolve, reject) => {
             client.on("sync", function syncListener(state) {
                 if (state === "SYNCING") {
@@ -255,11 +240,11 @@ describe("MatrixClient", function() {
                 },
             });
             httpLookups.push(FILTER_RESPONSE);
-            store.getFilterIdByName.andReturn(invalidFilterId);
+            store.getFilterIdByName.mockReturnValue(invalidFilterId);
 
             const filterName = getFilterName(client.credentials.userId);
             client.store.setFilterIdByName(filterName, invalidFilterId);
-            const filter = new sdk.Filter(client.credentials.userId);
+            const filter = new Filter(client.credentials.userId);
 
             client.getOrCreateFilter(filterName, filter).then(function(filterId) {
                 expect(filterId).toEqual(FILTER_RESPONSE.data.filter_id);
@@ -287,7 +272,7 @@ describe("MatrixClient", function() {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(2);
                     expect(client.retryImmediately()).toBe(true);
-                    clock.tick(1);
+                    jest.advanceTimersByTime(1);
                 } else if (state === "PREPARED" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -313,9 +298,9 @@ describe("MatrixClient", function() {
                     expect(client.retryImmediately()).toBe(
                         true, "retryImmediately returned false",
                     );
-                    clock.tick(1);
+                    jest.advanceTimersByTime(1);
                 } else if (state === "RECONNECTING" && httpLookups.length > 0) {
-                    clock.tick(10000);
+                    jest.advanceTimersByTime(10000);
                 } else if (state === "SYNCING" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -337,7 +322,7 @@ describe("MatrixClient", function() {
                 if (state === "ERROR" && httpLookups.length > 0) {
                     expect(httpLookups.length).toEqual(3);
                     expect(client.retryImmediately()).toBe(true);
-                    clock.tick(1);
+                    jest.advanceTimersByTime(1);
                 } else if (state === "PREPARED" && httpLookups.length === 0) {
                     client.removeListener("sync", syncListener);
                     done();
@@ -368,7 +353,7 @@ describe("MatrixClient", function() {
                     done();
                 }
                 // standard retry time is 5 to 10 seconds
-                clock.tick(10000);
+                jest.advanceTimersByTime(10000);
             };
         }
 
@@ -536,4 +521,19 @@ describe("MatrixClient", function() {
         xit("should be able to peek into a room using peekInRoom", function(done) {
         });
     });
+
+    describe("getPresence", function() {
+        it("should send a presence HTTP GET", function() {
+            httpLookups = [{
+                method: "GET",
+                path: `/presence/${encodeURIComponent(userId)}/status`,
+                data: {
+                    "presence": "unavailable",
+                    "last_active_ago": 420845,
+                },
+            }];
+            client.getPresence(userId);
+            expect(httpLookups.length).toEqual(0);
+        });
+    });
 });

spec/unit/pushprocessor.spec.js

@@ -1,9 +1,5 @@
-"use strict";
-import 'source-map-support/register';
-const PushProcessor = require("../../lib/pushprocessor");
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {PushProcessor} from "../../src/pushprocessor";
 
 describe('NotificationService', function() {
     const testUserId = "@ali:matrix.org";

spec/unit/realtime-callbacks.spec.js

@@ -1,53 +1,43 @@
-"use strict";
+import * as callbacks from "../../src/realtime-callbacks";
 
-import 'source-map-support/register';
-const callbacks = require("../../lib/realtime-callbacks");
-const testUtils = require("../test-utils.js");
-
-import expect from 'expect';
-import lolex from 'lolex';
+let wallTime = 1234567890;
+jest.useFakeTimers();
 
 describe("realtime-callbacks", function() {
-    let clock;
-
     function tick(millis) {
-        clock.tick(millis);
+        wallTime += millis;
+        jest.advanceTimersByTime(millis);
     }
 
     beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        clock = lolex.install();
-        const fakeDate = clock.Date;
-        callbacks.setNow(fakeDate.now.bind(fakeDate));
+        callbacks.setNow(() => wallTime);
     });
 
     afterEach(function() {
         callbacks.setNow();
-        clock.uninstall();
     });
 
     describe("setTimeout", function() {
         it("should call the callback after the timeout", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             callbacks.setTimeout(callback, 100);
 
-            expect(callback).toNotHaveBeenCalled();
+            expect(callback).not.toHaveBeenCalled();
             tick(100);
             expect(callback).toHaveBeenCalled();
         });
 
-
         it("should default to a zero timeout", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             callbacks.setTimeout(callback);
 
-            expect(callback).toNotHaveBeenCalled();
+            expect(callback).not.toHaveBeenCalled();
             tick(0);
             expect(callback).toHaveBeenCalled();
         });
 
         it("should pass any parameters to the callback", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             callbacks.setTimeout(callback, 0, "a", "b", "c");
             tick(0);
             expect(callback).toHaveBeenCalledWith("a", "b", "c");
@@ -56,8 +46,8 @@ describe("realtime-callbacks", function() {
         it("should set 'this' to the global object", function() {
             let passed = false;
             const callback = function() {
-                expect(this).toBe(global); // eslint-disable-line babel/no-invalid-this
-                expect(this.console).toBeTruthy(); // eslint-disable-line babel/no-invalid-this
+                expect(this).toBe(global); // eslint-disable-line @babel/no-invalid-this
+                expect(this.console).toBeTruthy(); // eslint-disable-line @babel/no-invalid-this
                 passed = true;
             };
             callbacks.setTimeout(callback);
@@ -66,10 +56,10 @@ describe("realtime-callbacks", function() {
         });
 
         it("should handle timeouts of several seconds", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             callbacks.setTimeout(callback, 2000);
 
-            expect(callback).toNotHaveBeenCalled();
+            expect(callback).not.toHaveBeenCalled();
             for (let i = 0; i < 4; i++) {
                 tick(500);
             }
@@ -77,24 +67,24 @@ describe("realtime-callbacks", function() {
         });
 
         it("should call multiple callbacks in the right order", function() {
-            const callback1 = expect.createSpy();
-            const callback2 = expect.createSpy();
-            const callback3 = expect.createSpy();
+            const callback1 = jest.fn();
+            const callback2 = jest.fn();
+            const callback3 = jest.fn();
             callbacks.setTimeout(callback2, 200);
             callbacks.setTimeout(callback1, 100);
             callbacks.setTimeout(callback3, 300);
 
-            expect(callback1).toNotHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
-            expect(callback3).toNotHaveBeenCalled();
+            expect(callback1).not.toHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
+            expect(callback3).not.toHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
-            expect(callback3).toNotHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
+            expect(callback3).not.toHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
-            expect(callback3).toNotHaveBeenCalled();
+            expect(callback3).not.toHaveBeenCalled();
             tick(100);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
@@ -102,35 +92,34 @@ describe("realtime-callbacks", function() {
         });
 
         it("should treat -ve timeouts the same as a zero timeout", function() {
-            const callback1 = expect.createSpy();
-            const callback2 = expect.createSpy();
+            const callback1 = jest.fn();
+            const callback2 = jest.fn();
 
             // check that cb1 is called before cb2
-            callback1.andCall(function() {
-                expect(callback2).toNotHaveBeenCalled();
+            callback1.mockImplementation(function() {
+                expect(callback2).not.toHaveBeenCalled();
             });
 
             callbacks.setTimeout(callback1);
             callbacks.setTimeout(callback2, -100);
 
-            expect(callback1).toNotHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback1).not.toHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
         });
 
         it("should not get confused by chained calls", function() {
-            const callback2 = expect.createSpy();
-            const callback1 = expect.createSpy();
-            callback1.andCall(function() {
+            const callback2 = jest.fn();
+            const callback1 = jest.fn(function() {
                 callbacks.setTimeout(callback2, 0);
-                expect(callback2).toNotHaveBeenCalled();
+                expect(callback2).not.toHaveBeenCalled();
             });
 
             callbacks.setTimeout(callback1);
-            expect(callback1).toNotHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback1).not.toHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             // the fake timer won't actually run callbacks registered during
@@ -140,16 +129,15 @@ describe("realtime-callbacks", function() {
         });
 
         it("should be immune to exceptions", function() {
-            const callback1 = expect.createSpy();
-            callback1.andCall(function() {
+            const callback1 = jest.fn(function() {
                 throw new Error("prepare to die");
             });
-            const callback2 = expect.createSpy();
+            const callback2 = jest.fn();
             callbacks.setTimeout(callback1, 0);
             callbacks.setTimeout(callback2, 0);
 
-            expect(callback1).toNotHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback1).not.toHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
             tick(0);
             expect(callback1).toHaveBeenCalled();
             expect(callback2).toHaveBeenCalled();
@@ -158,16 +146,16 @@ describe("realtime-callbacks", function() {
 
     describe("cancelTimeout", function() {
         it("should cancel a pending timeout", function() {
-            const callback = expect.createSpy();
+            const callback = jest.fn();
             const k = callbacks.setTimeout(callback);
             callbacks.clearTimeout(k);
             tick(0);
-            expect(callback).toNotHaveBeenCalled();
+            expect(callback).not.toHaveBeenCalled();
         });
 
         it("should not affect sooner timeouts", function() {
-            const callback1 = expect.createSpy();
-            const callback2 = expect.createSpy();
+            const callback1 = jest.fn();
+            const callback2 = jest.fn();
 
             callbacks.setTimeout(callback1, 100);
             const k = callbacks.setTimeout(callback2, 200);
@@ -175,10 +163,10 @@ describe("realtime-callbacks", function() {
 
             tick(100);
             expect(callback1).toHaveBeenCalled();
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
 
             tick(150);
-            expect(callback2).toNotHaveBeenCalled();
+            expect(callback2).not.toHaveBeenCalled();
         });
     });
 });

spec/unit/room-member.spec.js

@@ -1,10 +1,5 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const RoomMember = sdk.RoomMember;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {RoomMember} from "../../src/models/room-member";
 
 describe("RoomMember", function() {
     const roomId = "!foo:bar";
@@ -14,7 +9,6 @@ describe("RoomMember", function() {
     let member;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         member = new RoomMember(roomId, userA);
     });
 
@@ -36,13 +30,7 @@ describe("RoomMember", function() {
             const url = member.getAvatarUrl(hsUrl);
             // we don't care about how the mxc->http conversion is done, other
             // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
-        });
-
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.member event", function() {
-            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
+            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
         it("should return nothing if there is no m.room.member and allowDefault=false",
@@ -255,9 +243,9 @@ describe("RoomMember", function() {
             member.setMembershipEvent(joinEvent);
             expect(member.name).toEqual("Alice"); // prefer displayname
             member.setMembershipEvent(joinEvent, roomState);
-            expect(member.name).toNotEqual("Alice"); // it should disambig.
+            expect(member.name).not.toEqual("Alice"); // it should disambig.
             // user_id should be there somewhere
-            expect(member.name.indexOf(userA)).toNotEqual(-1);
+            expect(member.name.indexOf(userA)).not.toEqual(-1);
         });
 
         it("should emit 'RoomMember.membership' if the membership changes", function() {
@@ -328,9 +316,9 @@ describe("RoomMember", function() {
             };
             expect(member.name).toEqual(userA); // default = user_id
             member.setMembershipEvent(joinEvent, roomState);
-            expect(member.name).toNotEqual("Alíce"); // it should disambig.
+            expect(member.name).not.toEqual("Alíce"); // it should disambig.
             // user_id should be there somewhere
-            expect(member.name.indexOf(userA)).toNotEqual(-1);
+            expect(member.name.indexOf(userA)).not.toEqual(-1);
         });
     });
 });

spec/unit/room-state.spec.js

@@ -1,11 +1,6 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const RoomState = sdk.RoomState;
-const RoomMember = sdk.RoomMember;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {RoomState} from "../../src/models/room-state";
+import {RoomMember} from "../../src/models/room-member";
 
 describe("RoomState", function() {
     const roomId = "!foo:bar";
@@ -17,7 +12,6 @@ describe("RoomState", function() {
     let state;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         state = new RoomState(roomId);
         state.setStateEvents([
             utils.mkMembership({  // userA joined
@@ -49,8 +43,8 @@ describe("RoomState", function() {
             const members = state.getMembers();
             expect(members.length).toEqual(2);
             // ordering unimportant
-            expect([userA, userB].indexOf(members[0].userId)).toNotEqual(-1);
-            expect([userA, userB].indexOf(members[1].userId)).toNotEqual(-1);
+            expect([userA, userB].indexOf(members[0].userId)).not.toEqual(-1);
+            expect([userA, userB].indexOf(members[1].userId)).not.toEqual(-1);
         });
     });
 
@@ -120,8 +114,8 @@ describe("RoomState", function() {
             const events = state.getStateEvents("m.room.member");
             expect(events.length).toEqual(2);
             // ordering unimportant
-            expect([userA, userB].indexOf(events[0].getStateKey())).toNotEqual(-1);
-            expect([userA, userB].indexOf(events[1].getStateKey())).toNotEqual(-1);
+            expect([userA, userB].indexOf(events[0].getStateKey())).not.toEqual(-1);
+            expect([userA, userB].indexOf(events[1].getStateKey())).not.toEqual(-1);
         });
 
         it("should return a single MatrixEvent if a state_key was specified",
@@ -258,7 +252,7 @@ describe("RoomState", function() {
             });
             state.setStateEvents([memberEvent]);
 
-            expect(state.members[userA].setMembershipEvent).toNotHaveBeenCalled();
+            expect(state.members[userA].setMembershipEvent).not.toHaveBeenCalled();
             expect(state.members[userB].setMembershipEvent).toHaveBeenCalledWith(
                 memberEvent, state,
             );
@@ -306,7 +300,7 @@ describe("RoomState", function() {
             state.markOutOfBandMembersStarted();
             state.setOutOfBandMembers([oobMemberEvent]);
             const memberA = state.getMember(userA);
-            expect(memberA.events.member.getId()).toNotEqual(oobMemberEvent.getId());
+            expect(memberA.events.member.getId()).not.toEqual(oobMemberEvent.getId());
             expect(memberA.isOutOfBand()).toEqual(false);
         });
 

spec/unit/room.spec.js

@@ -1,14 +1,9 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const Room = sdk.Room;
-const RoomState = sdk.RoomState;
-const MatrixEvent = sdk.MatrixEvent;
-const EventStatus = sdk.EventStatus;
-const EventTimeline = sdk.EventTimeline;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../test-utils";
+import {EventStatus, MatrixEvent} from "../../src/models/event";
+import {EventTimeline} from "../../src/models/event-timeline";
+import {RoomState} from "../../src/models/room-state";
+import {Room} from "../../src/models/room";
+import {TestClient} from "../TestClient";
 
 describe("Room", function() {
     const roomId = "!foo:bar";
@@ -19,20 +14,19 @@ describe("Room", function() {
     let room;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         room = new Room(roomId);
         // mock RoomStates
         room.oldState = room.getLiveTimeline()._startState =
-            utils.mock(sdk.RoomState, "oldState");
+            utils.mock(RoomState, "oldState");
         room.currentState = room.getLiveTimeline()._endState =
-            utils.mock(sdk.RoomState, "currentState");
+            utils.mock(RoomState, "currentState");
     });
 
     describe("getAvatarUrl", function() {
         const hsUrl = "https://my.home.server";
 
         it("should return the URL from m.room.avatar preferentially", function() {
-            room.currentState.getStateEvents.andCall(function(type, key) {
+            room.currentState.getStateEvents.mockImplementation(function(type, key) {
                 if (type === "m.room.avatar" && key === "") {
                     return utils.mkEvent({
                         event: true,
@@ -49,13 +43,7 @@ describe("Room", function() {
             const url = room.getAvatarUrl(hsUrl);
             // we don't care about how the mxc->http conversion is done, other
             // than it contains the mxc body.
-            expect(url.indexOf("flibble/wibble")).toNotEqual(-1);
-        });
-
-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.avatar event", function() {
-            const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
+            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
         });
 
         it("should return nothing if there is no m.room.avatar and allowDefault=false",
@@ -67,13 +55,13 @@ describe("Room", function() {
 
     describe("getMember", function() {
         beforeEach(function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": {
                         userId: userA,
                         roomId: roomId,
                     },
-                }[userId];
+                }[userId] || null;
             });
         });
 
@@ -82,7 +70,7 @@ describe("Room", function() {
         });
 
         it("should return the member from current state", function() {
-            expect(room.getMember(userA)).toNotEqual(null);
+            expect(room.getMember(userA)).not.toEqual(null);
         });
     });
 
@@ -174,7 +162,7 @@ describe("Room", function() {
             );
             expect(events[0].forwardLooking).toBe(true);
             expect(events[1].forwardLooking).toBe(true);
-            expect(room.oldState.setStateEvents).toNotHaveBeenCalled();
+            expect(room.oldState.setStateEvents).not.toHaveBeenCalled();
         });
 
         it("should synthesize read receipts for the senders of events", function() {
@@ -183,7 +171,7 @@ describe("Room", function() {
                 membership: "join",
                 name: "Alice",
             };
-            room.currentState.getSentinelMember.andCall(function(uid) {
+            room.currentState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
@@ -292,13 +280,13 @@ describe("Room", function() {
                 membership: "join",
                 name: "Old Alice",
             };
-            room.currentState.getSentinelMember.andCall(function(uid) {
+            room.currentState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
                 return null;
             });
-            room.oldState.getSentinelMember.andCall(function(uid) {
+            room.oldState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return oldSentinel;
                 }
@@ -331,13 +319,13 @@ describe("Room", function() {
                 membership: "join",
                 name: "Old Alice",
             };
-            room.currentState.getSentinelMember.andCall(function(uid) {
+            room.currentState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return sentinel;
                 }
                 return null;
             });
-            room.oldState.getSentinelMember.andCall(function(uid) {
+            room.oldState.getSentinelMember.mockImplementation(function(uid) {
                 if (uid === userA) {
                     return oldSentinel;
                 }
@@ -379,7 +367,7 @@ describe("Room", function() {
             );
             expect(events[0].forwardLooking).toBe(false);
             expect(events[1].forwardLooking).toBe(false);
-            expect(room.currentState.setStateEvents).toNotHaveBeenCalled();
+            expect(room.currentState.setStateEvents).not.toHaveBeenCalled();
         });
     });
 
@@ -545,7 +533,7 @@ describe("Room", function() {
 
     describe("getJoinedMembers", function() {
         it("should return members whose membership is 'join'", function() {
-            room.currentState.getMembers.andCall(function() {
+            room.currentState.getMembers.mockImplementation(function() {
                 return [
                     { userId: "@alice:bar", membership: "join" },
                     { userId: "@bob:bar", membership: "invite" },
@@ -558,7 +546,7 @@ describe("Room", function() {
         });
 
         it("should return an empty list if no membership is 'join'", function() {
-            room.currentState.getMembers.andCall(function() {
+            room.currentState.getMembers.mockImplementation(function() {
                 return [
                     { userId: "@bob:bar", membership: "invite" },
                 ];
@@ -571,7 +559,7 @@ describe("Room", function() {
     describe("hasMembershipState", function() {
         it("should return true for a matching userId and membership",
         function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": { userId: "@alice:bar", membership: "join" },
                     "@bob:bar": { userId: "@bob:bar", membership: "invite" },
@@ -582,7 +570,7 @@ describe("Room", function() {
 
         it("should return false if match membership but no match userId",
         function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": { userId: "@alice:bar", membership: "join" },
                 }[userId];
@@ -592,7 +580,7 @@ describe("Room", function() {
 
         it("should return false if match userId but no match membership",
         function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": { userId: "@alice:bar", membership: "join" },
                 }[userId];
@@ -602,7 +590,7 @@ describe("Room", function() {
 
         it("should return false if no match membership or userId",
         function() {
-            room.currentState.getMember.andCall(function(userId) {
+            room.currentState.getMember.mockImplementation(function(userId) {
                 return {
                     "@alice:bar": { userId: "@alice:bar", membership: "join" },
                 }[userId];
@@ -624,13 +612,10 @@ describe("Room", function() {
                 }, event: true,
             })]);
         };
-        const setAliases = function(aliases, stateKey) {
-            if (!stateKey) {
-                stateKey = "flibble";
-            }
+        const setAltAliases = function(aliases) {
             room.addLiveEvents([utils.mkEvent({
-                type: "m.room.aliases", room: roomId, skey: stateKey, content: {
-                    aliases: aliases,
+                type: "m.room.canonical_alias", room: roomId, skey: "", content: {
+                    alt_aliases: aliases,
                 }, event: true,
             })]);
         };
@@ -814,8 +799,8 @@ describe("Room", function() {
                 addMember(userC);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
-                expect(name.indexOf(userC)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userC)).not.toEqual(-1, name);
             });
 
             it("should return the names of members in a public (public join_rules)" +
@@ -827,8 +812,8 @@ describe("Room", function() {
                 addMember(userC);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
-                expect(name.indexOf(userC)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
+                expect(name.indexOf(userC)).not.toEqual(-1, name);
             });
 
             it("should show the other user's name for public (public join_rules)" +
@@ -839,7 +824,7 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
             });
 
             it("should show the other user's name for private " +
@@ -850,7 +835,7 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
             });
 
             it("should show the other user's name for private" +
@@ -860,14 +845,14 @@ describe("Room", function() {
                 addMember(userB);
                 room.recalculate();
                 const name = room.name;
-                expect(name.indexOf(userB)).toNotEqual(-1, name);
+                expect(name.indexOf(userB)).not.toEqual(-1, name);
             });
 
             it("should show the room alias if one exists for private " +
             "(invite join_rules) rooms if a room name doesn't exist.", function() {
                 const alias = "#room_alias:here";
                 setJoinRule("invite");
-                setAliases([alias, "#another:one"]);
+                setAltAliases([alias, "#another:here"]);
                 room.recalculate();
                 const name = room.name;
                 expect(name).toEqual(alias);
@@ -877,7 +862,7 @@ describe("Room", function() {
             "(public join_rules) rooms if a room name doesn't exist.", function() {
                 const alias = "#room_alias:here";
                 setJoinRule("public");
-                setAliases([alias, "#another:one"]);
+                setAltAliases([alias, "#another:here"]);
                 room.recalculate();
                 const name = room.name;
                 expect(name).toEqual(alias);
@@ -1004,7 +989,7 @@ describe("Room", function() {
 
             it("should emit an event when a receipt is added",
             function() {
-                const listener = expect.createSpy();
+                const listener = jest.fn();
                 room.on("Room.receipt", listener);
 
                 const ts = 13787898424;
@@ -1175,7 +1160,7 @@ describe("Room", function() {
             it("should emit Room.tags event when new tags are " +
                "received on the event stream",
             function() {
-                const listener = expect.createSpy();
+                const listener = jest.fn();
                 room.on("Room.tags", listener);
 
                 const tags = { "m.foo": { "order": 0.5 } };
@@ -1192,7 +1177,10 @@ describe("Room", function() {
     describe("addPendingEvent", function() {
         it("should add pending events to the pendingEventList if " +
                       "pendingEventOrdering == 'detached'", function() {
-            const room = new Room(roomId, null, userA, {
+            const client = (new TestClient(
+                "@alice:example.com", "alicedevice",
+            )).client;
+            const room = new Room(roomId, client, userA, {
                 pendingEventOrdering: "detached",
             });
             const eventA = utils.mkMessage({
@@ -1242,7 +1230,10 @@ describe("Room", function() {
 
     describe("updatePendingEvent", function() {
         it("should remove cancelled events from the pending list", function() {
-            const room = new Room(roomId, null, userA, {
+            const client = (new TestClient(
+                "@alice:example.com", "alicedevice",
+            )).client;
+            const room = new Room(roomId, client, userA, {
                 pendingEventOrdering: "detached",
             });
             const eventA = utils.mkMessage({
@@ -1389,7 +1380,7 @@ describe("Room", function() {
             let hasThrown = false;
             try {
                 await room.loadMembersIfNeeded();
-            } catch(err) {
+            } catch (err) {
                 hasThrown = true;
             }
             expect(hasThrown).toEqual(true);

spec/unit/scheduler.spec.js

@@ -1,22 +1,18 @@
 // This file had a function whose name is all caps, which displeases eslint
 /* eslint new-cap: "off" */
 
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const MatrixScheduler = sdk.MatrixScheduler;
-const MatrixError = sdk.MatrixError;
-const utils = require("../test-utils");
+import {defer} from '../../src/utils';
+import {MatrixError} from "../../src/http-api";
+import {MatrixScheduler} from "../../src/scheduler";
+import * as utils from "../test-utils";
 
-import expect from 'expect';
-import lolex from 'lolex';
+jest.useFakeTimers();
 
 describe("MatrixScheduler", function() {
-    let clock;
     let scheduler;
     let retryFn;
     let queueFn;
-    let defer;
+    let deferred;
     const roomId = "!foo:bar";
     const eventA = utils.mkMessage({
         user: "@alice:bar", room: roomId, event: true,
@@ -26,8 +22,6 @@ describe("MatrixScheduler", function() {
     });
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-        clock = lolex.install();
         scheduler = new MatrixScheduler(function(ev, attempts, err) {
             if (retryFn) {
                 return retryFn(ev, attempts, err);
@@ -41,11 +35,7 @@ describe("MatrixScheduler", function() {
         });
         retryFn = null;
         queueFn = null;
-        defer = Promise.defer();
-    });
-
-    afterEach(function() {
-        clock.uninstall();
+        deferred = defer();
     });
 
     it("should process events in a queue in a FIFO manner", async function() {
@@ -55,8 +45,8 @@ describe("MatrixScheduler", function() {
         queueFn = function() {
             return "one_big_queue";
         };
-        const deferA = Promise.defer();
-        const deferB = Promise.defer();
+        const deferA = defer();
+        const deferB = defer();
         let yieldedA = false;
         scheduler.setProcessFunction(function(event) {
             if (yieldedA) {
@@ -82,7 +72,7 @@ describe("MatrixScheduler", function() {
     it("should invoke the retryFn on failure and wait the amount of time specified",
     async function() {
         const waitTimeMs = 1500;
-        const retryDefer = Promise.defer();
+        const retryDefer = defer();
         retryFn = function() {
             retryDefer.resolve();
             return waitTimeMs;
@@ -96,9 +86,9 @@ describe("MatrixScheduler", function() {
             procCount += 1;
             if (procCount === 1) {
                 expect(ev).toEqual(eventA);
-                return defer.promise;
+                return deferred.promise;
             } else if (procCount === 2) {
-                // don't care about this defer
+                // don't care about this deferred
                 return new Promise();
             }
             expect(procCount).toBeLessThan(3);
@@ -109,10 +99,10 @@ describe("MatrixScheduler", function() {
         // wait just long enough before it does
         await Promise.resolve();
         expect(procCount).toEqual(1);
-        defer.reject({});
+        deferred.reject({});
         await retryDefer.promise;
         expect(procCount).toEqual(1);
-        clock.tick(waitTimeMs);
+        jest.advanceTimersByTime(waitTimeMs);
         await Promise.resolve();
         expect(procCount).toEqual(2);
     });
@@ -129,8 +119,8 @@ describe("MatrixScheduler", function() {
             return "yep";
         };
 
-        const deferA = Promise.defer();
-        const deferB = Promise.defer();
+        const deferA = defer();
+        const deferB = defer();
         let procCount = 0;
         scheduler.setProcessFunction(function(ev) {
             procCount += 1;
@@ -153,7 +143,7 @@ describe("MatrixScheduler", function() {
         deferA.reject({});
         try {
             await globalA;
-        } catch(err) {
+        } catch (err) {
             await Promise.resolve();
             expect(procCount).toEqual(2);
         }
@@ -185,14 +175,14 @@ describe("MatrixScheduler", function() {
         const expectOrder = [
             eventA.getId(), eventB.getId(), eventD.getId(),
         ];
-        const deferA = Promise.defer();
+        const deferA = defer();
         scheduler.setProcessFunction(function(event) {
             const id = expectOrder.shift();
             expect(id).toEqual(event.getId());
             if (expectOrder.length === 0) {
                 done();
             }
-            return id === eventA.getId() ? deferA.promise : defer.promise;
+            return id === eventA.getId() ? deferA.promise : deferred.promise;
         });
         scheduler.queueEvent(eventA);
         scheduler.queueEvent(eventB);
@@ -203,7 +193,7 @@ describe("MatrixScheduler", function() {
         setTimeout(function() {
             deferA.resolve({});
         }, 1000);
-        clock.tick(1000);
+        jest.advanceTimersByTime(1000);
     });
 
     describe("queueEvent", function() {
@@ -306,7 +296,7 @@ describe("MatrixScheduler", function() {
             scheduler.setProcessFunction(function(ev) {
                 procCount += 1;
                 expect(ev).toEqual(eventA);
-                return defer.promise;
+                return deferred.promise;
             });
             // as queueing doesn't start processing synchronously anymore (see commit bbdb5ac)
             // wait just long enough before it does
@@ -322,7 +312,7 @@ describe("MatrixScheduler", function() {
             let procCount = 0;
             scheduler.setProcessFunction(function(ev) {
                 procCount += 1;
-                return defer.promise;
+                return deferred.promise;
             });
             expect(procCount).toEqual(0);
         });

spec/unit/sync-accumulator.spec.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2017 Vector Creations Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,19 +15,45 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-"use strict";
-import 'source-map-support/register';
-import utils from "../test-utils";
-import sdk from "../..";
-import expect from 'expect';
+import {SyncAccumulator} from "../../src/sync-accumulator";
 
-const SyncAccumulator = sdk.SyncAccumulator;
+// The event body & unsigned object get frozen to assert that they don't get altered
+// by the impl
+const RES_WITH_AGE = {
+    next_batch: "abc",
+    rooms: {
+        invite: {},
+        leave: {},
+        join: {
+            "!foo:bar": {
+                account_data: { events: [] },
+                ephemeral: { events: [] },
+                unread_notifications: {},
+                timeline: {
+                    events: [
+                        Object.freeze({
+                            content: {
+                                body: "This thing is happening right now!",
+                            },
+                            origin_server_ts: 123456789,
+                            sender: "@alice:localhost",
+                            type: "m.room.message",
+                            unsigned: Object.freeze({
+                                age: 50,
+                            }),
+                        }),
+                    ],
+                    prev_batch: "something",
+                },
+            },
+        },
+    },
+};
 
 describe("SyncAccumulator", function() {
     let sa;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         sa = new SyncAccumulator({
             maxTimelineEntries: 10,
         });
@@ -374,6 +401,39 @@ describe("SyncAccumulator", function() {
             expect(summary["m.joined_member_count"]).toEqual(5);
             expect(summary["m.heroes"]).toEqual(["@bob:bar"]);
         });
+
+        it("should return correctly adjusted age attributes", () => {
+            const delta = 1000;
+            const startingTs = 1000;
+
+            const oldDateNow = Date.now;
+            try {
+                Date.now = jest.fn();
+                Date.now.mockReturnValue(startingTs);
+
+                sa.accumulate(RES_WITH_AGE);
+
+                Date.now.mockReturnValue(startingTs + delta);
+
+                const output = sa.getJSON();
+                expect(output.roomsData.join["!foo:bar"].timeline.events[0].unsigned.age).toEqual(
+                    RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0].unsigned.age + delta,
+                );
+                expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                    Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+                );
+            } finally {
+                Date.now = oldDateNow;
+            }
+        });
+
+        it("should mangle age without adding extra keys", () => {
+            sa.accumulate(RES_WITH_AGE);
+            const output = sa.getJSON();
+            expect(Object.keys(output.roomsData.join["!foo:bar"].timeline.events[0])).toEqual(
+                Object.keys(RES_WITH_AGE.rooms.join["!foo:bar"].timeline.events[0]),
+            );
+        });
     });
 });
 

spec/unit/timeline-window.spec.js

@@ -1,13 +1,6 @@
-"use strict";
-import 'source-map-support/register';
-import Promise from 'bluebird';
-const sdk = require("../..");
-const EventTimeline = sdk.EventTimeline;
-const TimelineWindow = sdk.TimelineWindow;
-const TimelineIndex = require("../../lib/timeline-window").TimelineIndex;
-
-const utils = require("../test-utils");
-import expect from 'expect';
+import {EventTimeline} from "../../src/models/event-timeline";
+import {TimelineIndex, TimelineWindow} from "../../src/timeline-window";
+import * as utils from "../test-utils";
 
 const ROOM_ID = "roomId";
 const USER_ID = "userId";
@@ -67,10 +60,6 @@ function createLinkedTimelines() {
 
 
 describe("TimelineIndex", function() {
-    beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-    });
-
     describe("minIndex", function() {
         it("should return the min index relative to BaseIndex", function() {
             const timelineIndex = new TimelineIndex(createTimeline(), 0);
@@ -153,7 +142,7 @@ describe("TimelineWindow", function() {
     let timelineSet;
     let client;
     function createWindow(timeline, opts) {
-        timelineSet = {};
+        timelineSet = {getTimelineForEvent: () => null};
         client = {};
         client.getEventTimeline = function(timelineSet0, eventId0) {
             expect(timelineSet0).toBe(timelineSet);
@@ -163,12 +152,8 @@ describe("TimelineWindow", function() {
         return new TimelineWindow(client, timelineSet, opts);
     }
 
-    beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-    });
-
     describe("load", function() {
-        it("should initialise from the live timeline", function(done) {
+        it("should initialise from the live timeline", function() {
             const liveTimeline = createTimeline();
             const room = {};
             room.getLiveTimeline = function() {
@@ -176,17 +161,17 @@ describe("TimelineWindow", function() {
             };
 
             const timelineWindow = new TimelineWindow(undefined, room);
-            timelineWindow.load(undefined, 2).then(function() {
+            return timelineWindow.load(undefined, 2).then(function() {
                 const expectedEvents = liveTimeline.getEvents().slice(1);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).nodeify(done);
+            });
         });
 
-        it("should initialise from a specific event", function(done) {
+        it("should initialise from a specific event", function() {
             const timeline = createTimeline();
             const eventId = timeline.getEvents()[1].getId();
 
-            const timelineSet = {};
+            const timelineSet = {getTimelineForEvent: () => null};
             const client = {};
             client.getEventTimeline = function(timelineSet0, eventId0) {
                 expect(timelineSet0).toBe(timelineSet);
@@ -195,21 +180,20 @@ describe("TimelineWindow", function() {
             };
 
             const timelineWindow = new TimelineWindow(client, timelineSet);
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).nodeify(done);
+            });
         });
 
-        it("canPaginate should return false until load has returned",
-           function(done) {
+        it("canPaginate should return false until load has returned", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok1", EventTimeline.BACKWARDS);
             timeline.setPaginationToken("toktok2", EventTimeline.FORWARDS);
 
             const eventId = timeline.getEvents()[1].getId();
 
-            const timelineSet = {};
+            const timelineSet = {getTimelineForEvent: () => null};
             const client = {};
 
             const timelineWindow = new TimelineWindow(client, timelineSet);
@@ -222,25 +206,24 @@ describe("TimelineWindow", function() {
                 return Promise.resolve(timeline);
             };
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
                 expect(timelineWindow.canPaginate(EventTimeline.BACKWARDS))
                     .toBe(true);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(true);
-            }).nodeify(done);
+            });
         });
     });
 
     describe("pagination", function() {
-        it("should be able to advance across the initial timeline",
-           function(done) {
+        it("should be able to advance across the initial timeline", function() {
             const timeline = createTimeline();
             const eventId = timeline.getEvents()[1].getId();
             const timelineWindow = createWindow(timeline);
 
-            timelineWindow.load(eventId, 1).then(function() {
+            return timelineWindow.load(eventId, 1).then(function() {
                 const expectedEvents = [timeline.getEvents()[1]];
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -277,15 +260,15 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).nodeify(done);
+            });
         });
 
-        it("should advance into next timeline", function(done) {
+        it("should advance into next timeline", function() {
             const tls = createLinkedTimelines();
             const eventId = tls[0].getEvents()[1].getId();
             const timelineWindow = createWindow(tls[0], {windowLimit: 5});
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = tls[0].getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -322,15 +305,15 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.FORWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).nodeify(done);
+            });
         });
 
-        it("should retreat into previous timeline", function(done) {
+        it("should retreat into previous timeline", function() {
             const tls = createLinkedTimelines();
             const eventId = tls[1].getEvents()[1].getId();
             const timelineWindow = createWindow(tls[1], {windowLimit: 5});
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = tls[1].getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -367,10 +350,10 @@ describe("TimelineWindow", function() {
                 return timelineWindow.paginate(EventTimeline.BACKWARDS, 2);
             }).then(function(success) {
                 expect(success).toBe(false);
-            }).nodeify(done);
+            });
         });
 
-        it("should make forward pagination requests", function(done) {
+        it("should make forward pagination requests", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
 
@@ -386,7 +369,7 @@ describe("TimelineWindow", function() {
                 return Promise.resolve(true);
             };
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -399,11 +382,11 @@ describe("TimelineWindow", function() {
                 expect(success).toBe(true);
                 const expectedEvents = timeline.getEvents().slice(0, 5);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).nodeify(done);
+            });
         });
 
 
-        it("should make backward pagination requests", function(done) {
+        it("should make backward pagination requests", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.BACKWARDS);
 
@@ -419,7 +402,7 @@ describe("TimelineWindow", function() {
                 return Promise.resolve(true);
             };
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -432,11 +415,10 @@ describe("TimelineWindow", function() {
                 expect(success).toBe(true);
                 const expectedEvents = timeline.getEvents().slice(1, 6);
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
-            }).nodeify(done);
+            });
         });
 
-        it("should limit the number of unsuccessful pagination requests",
-        function(done) {
+        it("should limit the number of unsuccessful pagination requests", function() {
             const timeline = createTimeline();
             timeline.setPaginationToken("toktok", EventTimeline.FORWARDS);
 
@@ -452,7 +434,7 @@ describe("TimelineWindow", function() {
                 return Promise.resolve(true);
             };
 
-            timelineWindow.load(eventId, 3).then(function() {
+            return timelineWindow.load(eventId, 3).then(function() {
                 const expectedEvents = timeline.getEvents();
                 expect(timelineWindow.getEvents()).toEqual(expectedEvents);
 
@@ -471,7 +453,7 @@ describe("TimelineWindow", function() {
                     .toBe(false);
                 expect(timelineWindow.canPaginate(EventTimeline.FORWARDS))
                     .toBe(true);
-            }).nodeify(done);
+            });
         });
     });
 });

spec/unit/user.spec.js

@@ -1,17 +1,11 @@
-"use strict";
-import 'source-map-support/register';
-const sdk = require("../..");
-const User = sdk.User;
-const utils = require("../test-utils");
-
-import expect from 'expect';
+import {User} from "../../src/models/user";
+import * as utils from "../test-utils";
 
 describe("User", function() {
     const userId = "@alice:bar";
     let user;
 
     beforeEach(function() {
-        utils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
         user = new User(userId);
     });
 

spec/unit/utils.spec.js

@@ -1,15 +1,6 @@
-"use strict";
-import 'source-map-support/register';
-const utils = require("../../lib/utils");
-const testUtils = require("../test-utils");
-
-import expect from 'expect';
+import * as utils from "../../src/utils";
 
 describe("utils", function() {
-    beforeEach(function() {
-        testUtils.beforeEach(this); // eslint-disable-line babel/no-invalid-this
-    });
-
     describe("encodeParams", function() {
         it("should url encode and concat with &s", function() {
             const params = {
@@ -135,7 +126,7 @@ describe("utils", function() {
                 utils.checkObjectHasKeys({
                     foo: "bar",
                 }, ["foo"]);
-            }).toNotThrow();
+            }).not.toThrow();
         });
     });
 
@@ -152,7 +143,7 @@ describe("utils", function() {
                 utils.checkObjectHasNoAdditionalKeys({
                             foo: "bar",
                         }, ["foo"]);
-            }).toNotThrow();
+            }).not.toThrow();
         });
     });
 
@@ -291,4 +282,30 @@ describe("utils", function() {
             expect(target.nonenumerableProp).toBe(undefined);
         });
     });
+
+    describe("chunkPromises", function() {
+        it("should execute promises in chunks", async function() {
+            let promiseCount = 0;
+
+            function fn1() {
+                return new Promise(async function(resolve, reject) {
+                    await utils.sleep(1);
+                    expect(promiseCount).toEqual(0);
+                    ++promiseCount;
+                    resolve();
+                });
+            }
+
+            function fn2() {
+                return new Promise(function(resolve, reject) {
+                    expect(promiseCount).toEqual(1);
+                    ++promiseCount;
+                    resolve();
+                });
+            }
+
+            await utils.chunkPromises([fn1, fn2], 1);
+            expect(promiseCount).toEqual(2);
+        });
+    });
 });

spec/unit/webrtc/call.spec.ts

@@ -0,0 +1,257 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {TestClient} from '../../TestClient';
+import {MatrixCall, CallErrorCode} from '../../../src/webrtc/call';
+
+const DUMMY_SDP = (
+    "v=0\r\n" +
+    "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" +
+    "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" +
+    "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" +
+    "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" +
+    "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" +
+    "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" +
+    "a=ice-options:trickle\r\n" +
+    "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" +
+        "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" +
+    "a=setup:active\r\n" +
+    "a=mid:0\r\n" +
+    "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" +
+    "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" +
+    "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" +
+    "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" +
+    "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" +
+    "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" +
+    "a=sendrecv\r\n" +
+    "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" +
+    "a=rtcp-mux\r\n" +
+    "a=rtpmap:111 opus/48000/2\r\n" +
+    "a=rtcp-fb:111 transport-cc\r\n" +
+    "a=fmtp:111 minptime=10;useinbandfec=1\r\n" +
+    "a=rtpmap:103 ISAC/16000\r\n" +
+    "a=rtpmap:104 ISAC/32000\r\n" +
+    "a=rtpmap:9 G722/8000\r\n" +
+    "a=rtpmap:0 PCMU/8000\r\n" +
+    "a=rtpmap:8 PCMA/8000\r\n" +
+    "a=rtpmap:106 CN/32000\r\n" +
+    "a=rtpmap:105 CN/16000\r\n" +
+    "a=rtpmap:13 CN/8000\r\n" +
+    "a=rtpmap:110 telephone-event/48000\r\n" +
+    "a=rtpmap:112 telephone-event/32000\r\n" +
+    "a=rtpmap:113 telephone-event/16000\r\n" +
+    "a=rtpmap:126 telephone-event/8000\r\n" +
+    "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n"
+);
+
+class MockRTCPeerConnection {
+    localDescription: RTCSessionDescription;
+
+    constructor() {
+        this.localDescription = {
+            sdp: DUMMY_SDP,
+            type: 'offer',
+            toJSON: function() {},
+        };
+    }
+
+    addEventListener() {}
+    createOffer() {
+        return Promise.resolve({});
+    }
+    setRemoteDescription() {
+        return Promise.resolve();
+    }
+    setLocalDescription() {
+        return Promise.resolve();
+    }
+    close() {}
+    getStats() { return []; }
+}
+
+describe('Call', function() {
+    let client;
+    let call;
+    let prevNavigator;
+    let prevDocument;
+    let prevWindow;
+
+    beforeEach(function() {
+        prevNavigator = global.navigator;
+        prevDocument = global.document;
+        prevWindow = global.window;
+
+        global.navigator = {
+            mediaDevices: {
+                // @ts-ignore Mock
+                getUserMedia: () => {
+                    return {
+                        getTracks: () => [],
+                        getAudioTracks: () => [],
+                        getVideoTracks: () => [],
+                    };
+                },
+            },
+        };
+
+        global.window = {
+            // @ts-ignore Mock
+            RTCPeerConnection: MockRTCPeerConnection,
+            // @ts-ignore Mock
+            RTCSessionDescription: {},
+            // @ts-ignore Mock
+            RTCIceCandidate: {},
+            getUserMedia: {},
+        };
+        // @ts-ignore Mock
+        global.document = {};
+
+        client = new TestClient("@alice:foo", "somedevice", "token", undefined, {});
+        // We just stub out sendEvent: we're not interested in testing the client's
+        // event sending code here
+        client.client.sendEvent = () => {};
+        client.httpBackend.when("GET", "/voip/turnServer").respond(200, {});
+        call = new MatrixCall({
+            client: client.client,
+            roomId: '!foo:bar',
+        });
+        // call checks one of these is wired up
+        call.on('error', () => {});
+    });
+
+    afterEach(function() {
+        client.stop();
+        global.navigator = prevNavigator;
+        global.window = prevWindow;
+        global.document = prevDocument;
+    });
+
+    it('should ignore candidate events from non-matching party ID', async function() {
+        const callPromise = call.placeVoiceCall();
+        await client.httpBackend.flush();
+        await callPromise;
+        await call.onAnswerReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    answer: {
+                        sdp: DUMMY_SDP,
+                    },
+                };
+            },
+        });
+
+        call.peerConn.addIceCandidate = jest.fn();
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'some_other_party_id',
+                    candidates: [
+                        {
+                            candidate: '',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+
+        // Hangup to stop timers
+        call.hangup(CallErrorCode.UserHangup, true);
+    });
+
+    it('should add candidates received before answer if party ID is correct', async function() {
+        const callPromise = call.placeVoiceCall();
+        await client.httpBackend.flush();
+        await callPromise;
+        call.peerConn.addIceCandidate = jest.fn();
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    candidates: [
+                        {
+                            candidate: 'the_correct_candidate',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+
+        call.onRemoteIceCandidatesReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'some_other_party_id',
+                    candidates: [
+                        {
+                            candidate: 'the_wrong_candidate',
+                            sdpMid: '',
+                        },
+                    ],
+                };
+            },
+        });
+
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(0);
+
+        await call.onAnswerReceived({
+            getContent: () => {
+                return {
+                    version: 1,
+                    call_id: call.callId,
+                    party_id: 'the_correct_party_id',
+                    answer: {
+                        sdp: DUMMY_SDP,
+                    },
+                };
+            },
+        });
+
+        expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1);
+        expect(call.peerConn.addIceCandidate).toHaveBeenCalledWith({
+            candidate: 'the_correct_candidate',
+            sdpMid: '',
+        });
+    });
+});

src/@types/event.ts

@@ -0,0 +1,100 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export enum EventType {
+    // Room state events
+    RoomCanonicalAlias = "m.room.canonical_alias",
+    RoomCreate = "m.room.create",
+    RoomJoinRules = "m.room.join_rules",
+    RoomMember = "m.room.member",
+    RoomThirdPartyInvite = "m.room.third_party_invite",
+    RoomPowerLevels = "m.room.power_levels",
+    RoomName = "m.room.name",
+    RoomTopic = "m.room.topic",
+    RoomAvatar = "m.room.avatar",
+    RoomPinnedEvents = "m.room.pinned_events",
+    RoomEncryption = "m.room.encryption",
+    RoomHistoryVisibility = "m.room.history_visibility",
+    RoomGuestAccess = "m.room.guest_access",
+    RoomServerAcl = "m.room.server_acl",
+    RoomTombstone = "m.room.tombstone",
+    /**
+     * @deprecated Should not be used.
+     */
+    RoomAliases = "m.room.aliases", // deprecated https://matrix.org/docs/spec/client_server/r0.6.1#historical-events
+
+    // Spaces MSC1772
+    SpaceChild = "org.matrix.msc1772.space.child",
+    SpaceParent = "org.matrix.msc1772.space.parent",
+
+    // Room timeline events
+    RoomRedaction = "m.room.redaction",
+    RoomMessage = "m.room.message",
+    RoomMessageEncrypted = "m.room.encrypted",
+    Sticker = "m.sticker",
+    CallInvite = "m.call.invite",
+    CallCandidates = "m.call.candidates",
+    CallAnswer = "m.call.answer",
+    CallHangup = "m.call.hangup",
+    CallReject = "m.call.reject",
+    CallSelectAnswer = "m.call.select_answer",
+    CallNegotiate = "m.call.negotiate",
+    CallReplaces = "m.call.replaces",
+    KeyVerificationRequest = "m.key.verification.request",
+    KeyVerificationStart = "m.key.verification.start",
+    KeyVerificationCancel = "m.key.verification.cancel",
+    KeyVerificationMac = "m.key.verification.mac",
+    KeyVerificationDone = "m.key.verification.done",
+    // use of this is discouraged https://matrix.org/docs/spec/client_server/r0.6.1#m-room-message-feedback
+    RoomMessageFeedback = "m.room.message.feedback",
+
+    // Room ephemeral events
+    Typing = "m.typing",
+    Receipt = "m.receipt",
+    Presence = "m.presence",
+
+    // Room account_data events
+    FullyRead = "m.fully_read",
+    Tag = "m.tag",
+
+    // User account_data events
+    PushRules = "m.push_rules",
+    Direct = "m.direct",
+    IgnoredUserList = "m.ignored_user_list",
+
+    // to_device events
+    RoomKey = "m.room_key",
+    RoomKeyRequest = "m.room_key_request",
+    ForwardedRoomKey = "m.forwarded_room_key",
+    Dummy = "m.dummy",
+}
+
+export enum MsgType {
+    Text = "m.text",
+    Emote = "m.emote",
+    Notice = "m.notice",
+    Image = "m.image",
+    File = "m.file",
+    Audio = "m.audio",
+    Location = "m.location",
+    Video = "m.video",
+}
+
+export const RoomCreateTypeField = "org.matrix.msc1772.type"; // Spaces MSC1772
+
+export enum RoomType {
+    Space = "org.matrix.msc1772.space", // Spaces MSC1772
+}

src/@types/global.d.ts

@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// this is needed to tell TS about global.Olm
+import * as Olm from "olm"; // eslint-disable-line @typescript-eslint/no-unused-vars
+
+export {};
+
+declare global {
+    namespace NodeJS {
+        interface Global {
+            localStorage: Storage;
+        }
+    }
+
+    interface Window {
+        electron?: Electron;
+    }
+
+    interface Electron {
+        getDesktopCapturerSources(options: GetSourcesOptions): Promise<Array<DesktopCapturerSource>>;
+    }
+
+    interface MediaDevices {
+        // This is experimental and types don't know about it yet
+        // https://github.com/microsoft/TypeScript/issues/33232
+        getDisplayMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise<MediaStream>;
+        getUserMedia(constraints: MediaStreamConstraints | DesktopCapturerConstraints): Promise<MediaStream>;
+    }
+
+    interface DesktopCapturerConstraints {
+        audio: boolean | {
+            mandatory: {
+                chromeMediaSource: string;
+                chromeMediaSourceId: string;
+            };
+        };
+        video: boolean | {
+            mandatory: {
+                chromeMediaSource: string;
+                chromeMediaSourceId: string;
+            };
+        };
+    }
+
+    interface DesktopCapturerSource {
+        id: string;
+        name: string;
+        thumbnailURL: string;
+    }
+
+    interface GetSourcesOptions {
+        types: Array<string>;
+        thumbnailSize?: {
+            height: number;
+            width: number;
+        };
+        fetchWindowIcons?: boolean;
+    }
+
+    interface HTMLAudioElement {
+        // sinkId & setSinkId are experimental and typescript doesn't know about them
+        sinkId: string;
+        setSinkId(outputId: string);
+    }
+
+    interface DummyInterfaceWeShouldntBeUsingThis {}
+
+    interface Navigator {
+        // We check for the webkit-prefixed getUserMedia to detect if we're
+        // on webkit: we should check if we still need to do this
+        webkitGetUserMedia: DummyInterfaceWeShouldntBeUsingThis;
+    }
+}

src/ReEmitter.js

@@ -1,52 +0,0 @@
-/*
-Copyright 2015, 2016 OpenMarket Ltd
-Copyright 2017 Vector Creations Ltd
-Copyright 2017 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-/**
- * @module
- */
-
-export default class Reemitter {
-    constructor(target) {
-        this.target = target;
-
-        // We keep one bound event handler for each event name so we know
-        // what event is arriving
-        this.boundHandlers = {};
-    }
-
-    _handleEvent(eventName, ...args) {
-        this.target.emit(eventName, ...args);
-    }
-
-    reEmit(source, eventNames) {
-        // We include the source as the last argument for event handlers which may need it,
-        // such as read receipt listeners on the client class which won't have the context
-        // of the room.
-        const forSource = (handler, ...args) => {
-            handler(...args, source);
-        };
-        for (const eventName of eventNames) {
-            if (this.boundHandlers[eventName] === undefined) {
-                this.boundHandlers[eventName] = this._handleEvent.bind(this, eventName);
-            }
-
-            const boundHandler = forSource.bind(this, this.boundHandlers[eventName]);
-            source.on(eventName, boundHandler);
-        }
-    }
-}

src/ReEmitter.ts

@@ -0,0 +1,50 @@
+/*
+Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2017 Vector Creations Ltd
+Copyright 2017 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+
+export class ReEmitter {
+    private target: EventEmitter;
+
+    constructor(target: EventEmitter) {
+        this.target = target;
+    }
+
+    reEmit(source: EventEmitter, eventNames: string[]) {
+        for (const eventName of eventNames) {
+            // We include the source as the last argument for event handlers which may need it,
+            // such as read receipt listeners on the client class which won't have the context
+            // of the room.
+            const forSource = (...args) => {
+                // EventEmitter special cases 'error' to make the emit function throw if no
+                // handler is attached, which sort of makes sense for making sure that something
+                // handles an error, but for re-emitting, there could be a listener on the original
+                // source object so the test doesn't really work. We *could* try to replicate the
+                // same logic and throw if there is no listener on either the source or the target,
+                // but this behaviour is fairly undesireable for us anyway: the main place we throw
+                // 'error' events is for calls, where error events are usually emitted some time
+                // later by a different part of the code where 'emit' throwing because the app hasn't
+                // added an error handler isn't terribly helpful. (A better fix in retrospect may
+                // have been to just avoid using the event name 'error', but backwards compat...)
+                if (eventName === 'error' && this.target.listenerCount('error') === 0) return;
+                this.target.emit(eventName, ...args, source);
+            };
+            source.on(eventName, forSource);
+        }
+    }
+}

src/autodiscovery.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,9 +17,8 @@ limitations under the License.
 
 /** @module auto-discovery */
 
-import Promise from 'bluebird';
-import logger from './logger';
-import { URL as NodeURL } from "url";
+import {logger} from './logger';
+import {URL as NodeURL} from "url";
 
 // Dev note: Auto discovery is part of the spec.
 // See: https://matrix.org/docs/spec/client_server/r0.4.0.html#server-discovery
@@ -275,21 +275,11 @@ export class AutoDiscovery {
         let isUrl = "";
         if (wellknown["m.identity_server"]) {
             // We prepare a failing identity server response to save lines later
-            // in this branch. Note that we also fail the homeserver check in the
-            // object because according to the spec we're supposed to FAIL_ERROR
-            // if *anything* goes wrong with the IS validation, including invalid
-            // format. This means we're supposed to stop discovery completely.
+            // in this branch.
             const failingClientConfig = {
-                "m.homeserver": {
-                    state: AutoDiscovery.FAIL_ERROR,
-                    error: AutoDiscovery.ERROR_INVALID_IS,
-
-                    // We'll provide the base_url that was previously valid for
-                    // debugging purposes.
-                    base_url: clientConfig["m.homeserver"].base_url,
-                },
+                "m.homeserver": clientConfig["m.homeserver"],
                 "m.identity_server": {
-                    state: AutoDiscovery.FAIL_ERROR,
+                    state: AutoDiscovery.FAIL_PROMPT,
                     error: AutoDiscovery.ERROR_INVALID_IS,
                     base_url: null,
                 },
@@ -429,6 +419,26 @@ export class AutoDiscovery {
         return AutoDiscovery.fromDiscoveryConfig(wellknown.raw);
     }
 
+    /**
+     * Gets the raw discovery client configuration for the given domain name.
+     * Should only be used if there's no validation to be done on the resulting
+     * object, otherwise use findClientConfig().
+     * @param {string} domain The domain to get the client config for.
+     * @returns {Promise<object>} Resolves to the domain's client config. Can
+     * be an empty object.
+     */
+    static async getRawClientConfig(domain) {
+        if (!domain || typeof(domain) !== "string" || domain.length === 0) {
+            throw new Error("'domain' must be a string of non-zero length");
+        }
+
+        const response = await this._fetchWellKnownObject(
+            `https://${domain}/.well-known/matrix/client`,
+        );
+        if (!response) return {};
+        return response.raw || {};
+    }
+
     /**
      * Sanitizes a given URL to ensure it is either an HTTP or HTTP URL and
      * is suitable for the requirements laid out by .well-known auto discovery.
@@ -492,10 +502,12 @@ export class AutoDiscovery {
             request(
                 { method: "GET", uri: url, timeout: 5000 },
                 (err, response, body) => {
-                    if (err || response.statusCode < 200 || response.statusCode >= 300) {
+                    if (err || response &&
+                        (response.statusCode < 200 || response.statusCode >= 300)
+                    ) {
                         let action = "FAIL_PROMPT";
                         let reason = (err ? err.message : null) || "General failure";
-                        if (response.statusCode === 404) {
+                        if (response && response.statusCode === 404) {
                             action = "IGNORE";
                             reason = AutoDiscovery.ERROR_MISSING_WELLKNOWN;
                         }

src/browser-index.js

@@ -0,0 +1,54 @@
+/*
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import * as matrixcs from "./matrix";
+import request from "browser-request";
+import queryString from "qs";
+
+matrixcs.request(function(opts, fn) {
+    // We manually fix the query string for browser-request because
+    // it doesn't correctly handle cases like ?via=one&via=two. Instead
+    // we mimic `request`'s query string interface to make it all work
+    // as expected.
+    // browser-request will happily take the constructed string as the
+    // query string without trying to modify it further.
+    opts.qs = queryString.stringify(opts.qs || {}, opts.qsStringifyOptions);
+    return request(opts, fn);
+});
+
+// just *accessing* indexedDB throws an exception in firefox with
+// indexeddb disabled.
+let indexedDB;
+try {
+    indexedDB = global.indexedDB;
+} catch (e) {}
+
+// if our browser (appears to) support indexeddb, use an indexeddb crypto store.
+if (indexedDB) {
+    matrixcs.setCryptoStoreFactory(
+        function() {
+            return new matrixcs.IndexedDBCryptoStore(
+                indexedDB, "matrix-js-sdk:crypto",
+            );
+        },
+    );
+}
+
+// We export 3 things to make browserify happy as well as downstream projects.
+// It's awkward, but required.
+export * from "./matrix";
+export default matrixcs; // keep export for browserify package deps
+global.matrixcs = matrixcs;

src/content-helpers.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,88 +14,86 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /** @module ContentHelpers */
-module.exports = {
-    /**
-     * Generates the content for a HTML Message event
-     * @param {string} body the plaintext body of the message
-     * @param {string} htmlBody the HTML representation of the message
-     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
-     */
-    makeHtmlMessage: function(body, htmlBody) {
-        return {
-            msgtype: "m.text",
-            format: "org.matrix.custom.html",
-            body: body,
-            formatted_body: htmlBody,
-        };
-    },
 
-    /**
-     * Generates the content for a HTML Notice event
-     * @param {string} body the plaintext body of the notice
-     * @param {string} htmlBody the HTML representation of the notice
-     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
-     */
-    makeHtmlNotice: function(body, htmlBody) {
-        return {
-            msgtype: "m.notice",
-            format: "org.matrix.custom.html",
-            body: body,
-            formatted_body: htmlBody,
-        };
-    },
+/**
+ * Generates the content for a HTML Message event
+ * @param {string} body the plaintext body of the message
+ * @param {string} htmlBody the HTML representation of the message
+ * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+ */
+export function makeHtmlMessage(body, htmlBody) {
+    return {
+        msgtype: "m.text",
+        format: "org.matrix.custom.html",
+        body: body,
+        formatted_body: htmlBody,
+    };
+}
 
-    /**
-     * Generates the content for a HTML Emote event
-     * @param {string} body the plaintext body of the emote
-     * @param {string} htmlBody the HTML representation of the emote
-     * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
-     */
-    makeHtmlEmote: function(body, htmlBody) {
-        return {
-            msgtype: "m.emote",
-            format: "org.matrix.custom.html",
-            body: body,
-            formatted_body: htmlBody,
-        };
-    },
+/**
+ * Generates the content for a HTML Notice event
+ * @param {string} body the plaintext body of the notice
+ * @param {string} htmlBody the HTML representation of the notice
+ * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+ */
+export function makeHtmlNotice(body, htmlBody) {
+    return {
+        msgtype: "m.notice",
+        format: "org.matrix.custom.html",
+        body: body,
+        formatted_body: htmlBody,
+    };
+}
 
-    /**
-     * Generates the content for a Plaintext Message event
-     * @param {string} body the plaintext body of the emote
-     * @returns {{msgtype: string, body: string}}
-     */
-    makeTextMessage: function(body) {
-        return {
-            msgtype: "m.text",
-            body: body,
-        };
-    },
+/**
+ * Generates the content for a HTML Emote event
+ * @param {string} body the plaintext body of the emote
+ * @param {string} htmlBody the HTML representation of the emote
+ * @returns {{msgtype: string, format: string, body: string, formatted_body: string}}
+ */
+export function makeHtmlEmote(body, htmlBody) {
+    return {
+        msgtype: "m.emote",
+        format: "org.matrix.custom.html",
+        body: body,
+        formatted_body: htmlBody,
+    };
+}
 
-    /**
-     * Generates the content for a Plaintext Notice event
-     * @param {string} body the plaintext body of the notice
-     * @returns {{msgtype: string, body: string}}
-     */
-    makeNotice: function(body) {
-        return {
-            msgtype: "m.notice",
-            body: body,
-        };
-    },
+/**
+ * Generates the content for a Plaintext Message event
+ * @param {string} body the plaintext body of the emote
+ * @returns {{msgtype: string, body: string}}
+ */
+export function makeTextMessage(body) {
+    return {
+        msgtype: "m.text",
+        body: body,
+    };
+}
 
-    /**
-     * Generates the content for a Plaintext Emote event
-     * @param {string} body the plaintext body of the emote
-     * @returns {{msgtype: string, body: string}}
-     */
-    makeEmoteMessage: function(body) {
-        return {
-            msgtype: "m.emote",
-            body: body,
-        };
-    },
-};
+/**
+ * Generates the content for a Plaintext Notice event
+ * @param {string} body the plaintext body of the notice
+ * @returns {{msgtype: string, body: string}}
+ */
+export function makeNotice(body) {
+    return {
+        msgtype: "m.notice",
+        body: body,
+    };
+}
+
+/**
+ * Generates the content for a Plaintext Emote event
+ * @param {string} body the plaintext body of the emote
+ * @returns {{msgtype: string, body: string}}
+ */
+export function makeEmoteMessage(body) {
+    return {
+        msgtype: "m.emote",
+        body: body,
+    };
+}

src/content-repo.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2015, 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,95 +17,61 @@ limitations under the License.
 /**
  * @module content-repo
  */
-const utils = require("./utils");
 
-/** Content Repo utility functions */
-module.exports = {
-    /**
-     * Get the HTTP URL for an MXC URI.
-     * @param {string} baseUrl The base homeserver url which has a content repo.
-     * @param {string} mxc The mxc:// URI.
-     * @param {Number} width The desired width of the thumbnail.
-     * @param {Number} height The desired height of the thumbnail.
-     * @param {string} resizeMethod The thumbnail resize method to use, either
-     * "crop" or "scale".
-     * @param {Boolean} allowDirectLinks If true, return any non-mxc URLs
-     * directly. Fetching such URLs will leak information about the user to
-     * anyone they share a room with. If false, will return the emptry string
-     * for such URLs.
-     * @return {string} The complete URL to the content.
-     */
-    getHttpUriForMxc: function(baseUrl, mxc, width, height,
-                               resizeMethod, allowDirectLinks) {
-        if (typeof mxc !== "string" || !mxc) {
+import * as utils from "./utils";
+
+/**
+ * Get the HTTP URL for an MXC URI.
+ * @param {string} baseUrl The base homeserver url which has a content repo.
+ * @param {string} mxc The mxc:// URI.
+ * @param {Number} width The desired width of the thumbnail.
+ * @param {Number} height The desired height of the thumbnail.
+ * @param {string} resizeMethod The thumbnail resize method to use, either
+ * "crop" or "scale".
+ * @param {Boolean} allowDirectLinks If true, return any non-mxc URLs
+ * directly. Fetching such URLs will leak information about the user to
+ * anyone they share a room with. If false, will return the emptry string
+ * for such URLs.
+ * @return {string} The complete URL to the content.
+ */
+export function getHttpUriForMxc(baseUrl, mxc, width, height,
+                           resizeMethod, allowDirectLinks) {
+    if (typeof mxc !== "string" || !mxc) {
+        return '';
+    }
+    if (mxc.indexOf("mxc://") !== 0) {
+        if (allowDirectLinks) {
+            return mxc;
+        } else {
             return '';
         }
-        if (mxc.indexOf("mxc://") !== 0) {
-            if (allowDirectLinks) {
-                return mxc;
-            } else {
-                return '';
-            }
-        }
-        let serverAndMediaId = mxc.slice(6); // strips mxc://
-        let prefix = "/_matrix/media/r0/download/";
-        const params = {};
+    }
+    let serverAndMediaId = mxc.slice(6); // strips mxc://
+    let prefix = "/_matrix/media/r0/download/";
+    const params = {};
 
-        if (width) {
-            params.width = Math.round(width);
-        }
-        if (height) {
-            params.height = Math.round(height);
-        }
-        if (resizeMethod) {
-            params.method = resizeMethod;
-        }
-        if (utils.keys(params).length > 0) {
-            // these are thumbnailing params so they probably want the
-            // thumbnailing API...
-            prefix = "/_matrix/media/r0/thumbnail/";
-        }
+    if (width) {
+        params.width = Math.round(width);
+    }
+    if (height) {
+        params.height = Math.round(height);
+    }
+    if (resizeMethod) {
+        params.method = resizeMethod;
+    }
+    if (utils.keys(params).length > 0) {
+        // these are thumbnailing params so they probably want the
+        // thumbnailing API...
+        prefix = "/_matrix/media/r0/thumbnail/";
+    }
 
-        const fragmentOffset = serverAndMediaId.indexOf("#");
-        let fragment = "";
-        if (fragmentOffset >= 0) {
-            fragment = serverAndMediaId.substr(fragmentOffset);
-            serverAndMediaId = serverAndMediaId.substr(0, fragmentOffset);
-        }
-        return baseUrl + prefix + serverAndMediaId +
-            (utils.keys(params).length === 0 ? "" :
-            ("?" + utils.encodeParams(params))) + fragment;
-    },
-
-    /**
-     * Get an identicon URL from an arbitrary string.
-     * @param {string} baseUrl The base homeserver url which has a content repo.
-     * @param {string} identiconString The string to create an identicon for.
-     * @param {Number} width The desired width of the image in pixels. Default: 96.
-     * @param {Number} height The desired height of the image in pixels. Default: 96.
-     * @return {string} The complete URL to the identicon.
-     * @deprecated This is no longer in the specification.
-     */
-    getIdenticonUri: function(baseUrl, identiconString, width, height) {
-        if (!identiconString) {
-            return null;
-        }
-        if (!width) {
-            width = 96;
-        }
-        if (!height) {
-            height = 96;
-        }
-        const params = {
-            width: width,
-            height: height,
-        };
-
-        const path = utils.encodeUri("/_matrix/media/unstable/identicon/$ident", {
-            $ident: identiconString,
-        });
-        return baseUrl + path +
-            (utils.keys(params).length === 0 ? "" :
-                ("?" + utils.encodeParams(params)));
-    },
-};
+    const fragmentOffset = serverAndMediaId.indexOf("#");
+    let fragment = "";
+    if (fragmentOffset >= 0) {
+        fragment = serverAndMediaId.substr(fragmentOffset);
+        serverAndMediaId = serverAndMediaId.substr(0, fragmentOffset);
+    }
+    return baseUrl + prefix + serverAndMediaId +
+        (utils.keys(params).length === 0 ? "" :
+        ("?" + utils.encodeParams(params))) + fragment;
+}

src/crypto/CrossSigning.js

@@ -0,0 +1,816 @@
+/*
+Copyright 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Cross signing methods
+ * @module crypto/CrossSigning
+ */
+
+import {decodeBase64, encodeBase64, pkSign, pkVerify} from './olmlib';
+import {EventEmitter} from 'events';
+import {logger} from '../logger';
+import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+
+const KEY_REQUEST_TIMEOUT_MS = 1000 * 60;
+
+function publicKeyFromKeyInfo(keyInfo) {
+    // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
+    // We assume only a single key, and we want the bare form without type
+    // prefix, so we select the values.
+    return Object.values(keyInfo.keys)[0];
+}
+
+export class CrossSigningInfo extends EventEmitter {
+    /**
+     * Information about a user's cross-signing keys
+     *
+     * @class
+     *
+     * @param {string} userId the user that the information is about
+     * @param {object} callbacks Callbacks used to interact with the app
+     *     Requires getCrossSigningKey and saveCrossSigningKeys
+     * @param {object} cacheCallbacks Callbacks used to interact with the cache
+     */
+    constructor(userId, callbacks, cacheCallbacks) {
+        super();
+
+        // you can't change the userId
+        Object.defineProperty(this, 'userId', {
+            enumerable: true,
+            value: userId,
+        });
+        this._callbacks = callbacks || {};
+        this._cacheCallbacks = cacheCallbacks || {};
+        this.keys = {};
+        this.firstUse = true;
+        // This tracks whether we've ever verified this user with any identity.
+        // When you verify a user, any devices online at the time that receive
+        // the verifying signature via the homeserver will latch this to true
+        // and can use it in the future to detect cases where the user has
+        // become unverifed later for any reason.
+        this.crossSigningVerifiedBefore = false;
+    }
+
+    static fromStorage(obj, userId) {
+        const res = new CrossSigningInfo(userId);
+        for (const prop in obj) {
+            if (obj.hasOwnProperty(prop)) {
+                res[prop] = obj[prop];
+            }
+        }
+        return res;
+    }
+
+    toStorage() {
+        return {
+            keys: this.keys,
+            firstUse: this.firstUse,
+            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
+        };
+    }
+
+    /**
+     * Calls the app callback to ask for a private key
+     *
+     * @param {string} type The key type ("master", "self_signing", or "user_signing")
+     * @param {string} expectedPubkey The matching public key or undefined to use
+     *     the stored public key for the given key type.
+     * @returns {Array} An array with [ public key, Olm.PkSigning ]
+     */
+    async getCrossSigningKey(type, expectedPubkey) {
+        const shouldCache = ["master", "self_signing", "user_signing"].indexOf(type) >= 0;
+
+        if (!this._callbacks.getCrossSigningKey) {
+            throw new Error("No getCrossSigningKey callback supplied");
+        }
+
+        if (expectedPubkey === undefined) {
+            expectedPubkey = this.getId(type);
+        }
+
+        function validateKey(key) {
+            if (!key) return;
+            const signing = new global.Olm.PkSigning();
+            const gotPubkey = signing.init_with_seed(key);
+            if (gotPubkey === expectedPubkey) {
+                return [gotPubkey, signing];
+            }
+            signing.free();
+        }
+
+        let privkey;
+        if (this._cacheCallbacks.getCrossSigningKeyCache && shouldCache) {
+            privkey = await this._cacheCallbacks
+              .getCrossSigningKeyCache(type, expectedPubkey);
+        }
+
+        const cacheresult = validateKey(privkey);
+        if (cacheresult) {
+            return cacheresult;
+        }
+
+        privkey = await this._callbacks.getCrossSigningKey(type, expectedPubkey);
+        const result = validateKey(privkey);
+        if (result) {
+            if (this._cacheCallbacks.storeCrossSigningKeyCache && shouldCache) {
+                await this._cacheCallbacks.storeCrossSigningKeyCache(type, privkey);
+            }
+            return result;
+        }
+
+        /* No keysource even returned a key */
+        if (!privkey) {
+            throw new Error(
+                "getCrossSigningKey callback for " + type + " returned falsey",
+            );
+        }
+
+        /* We got some keys from the keysource, but none of them were valid */
+        throw new Error(
+            "Key type " + type + " from getCrossSigningKey callback did not match",
+        );
+    }
+
+    /**
+     * Check whether the private keys exist in secret storage.
+     * XXX: This could be static, be we often seem to have an instance when we
+     * want to know this anyway...
+     *
+     * @param {SecretStorage} secretStorage The secret store using account data
+     * @returns {object} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
+     */
+    async isStoredInSecretStorage(secretStorage) {
+        // check what SSSS keys have encrypted the master key (if any)
+        const stored =
+              await secretStorage.isStored("m.cross_signing.master", false) || {};
+        // then check which of those SSSS keys have also encrypted the SSK and USK
+        function intersect(s) {
+            for (const k of Object.keys(stored)) {
+                if (!s[k]) {
+                    delete stored[k];
+                }
+            }
+        }
+        for (const type of ["self_signing", "user_signing"]) {
+            intersect(
+                await secretStorage.isStored(`m.cross_signing.${type}`, false) || {},
+            );
+        }
+        return Object.keys(stored).length ? stored : null;
+    }
+
+    /**
+     * Store private keys in secret storage for use by other devices. This is
+     * typically called in conjunction with the creation of new cross-signing
+     * keys.
+     *
+     * @param {Map} keys The keys to store
+     * @param {SecretStorage} secretStorage The secret store using account data
+     */
+    static async storeInSecretStorage(keys, secretStorage) {
+        for (const [type, privateKey] of keys) {
+            const encodedKey = encodeBase64(privateKey);
+            await secretStorage.store(`m.cross_signing.${type}`, encodedKey);
+        }
+    }
+
+    /**
+     * Get private keys from secret storage created by some other device. This
+     * also passes the private keys to the app-specific callback.
+     *
+     * @param {string} type The type of key to get.  One of "master",
+     * "self_signing", or "user_signing".
+     * @param {SecretStorage} secretStorage The secret store using account data
+     * @return {Uint8Array} The private key
+     */
+    static async getFromSecretStorage(type, secretStorage) {
+        const encodedKey = await secretStorage.get(`m.cross_signing.${type}`);
+        if (!encodedKey) {
+            return null;
+        }
+        return decodeBase64(encodedKey);
+    }
+
+    /**
+     * Check whether the private keys exist in the local key cache.
+     *
+     * @param {string} [type] The type of key to get. One of "master",
+     * "self_signing", or "user_signing". Optional, will check all by default.
+     * @returns {boolean} True if all keys are stored in the local cache.
+     */
+    async isStoredInKeyCache(type) {
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return false;
+        const types = type ? [type] : ["master", "self_signing", "user_signing"];
+        for (const t of types) {
+            if (!await cacheCallbacks.getCrossSigningKeyCache(t)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Get cross-signing private keys from the local cache.
+     *
+     * @returns {Map} A map from key type (string) to private key (Uint8Array)
+     */
+    async getCrossSigningKeysFromCache() {
+        const keys = new Map();
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return keys;
+        for (const type of ["master", "self_signing", "user_signing"]) {
+            const privKey = await cacheCallbacks.getCrossSigningKeyCache(type);
+            if (!privKey) {
+                continue;
+            }
+            keys.set(type, privKey);
+        }
+        return keys;
+    }
+
+    /**
+     * Get the ID used to identify the user. This can also be used to test for
+     * the existence of a given key type.
+     *
+     * @param {string} type The type of key to get the ID of.  One of "master",
+     * "self_signing", or "user_signing".  Defaults to "master".
+     *
+     * @return {string} the ID
+     */
+    getId(type) {
+        type = type || "master";
+        if (!this.keys[type]) return null;
+        const keyInfo = this.keys[type];
+        return publicKeyFromKeyInfo(keyInfo);
+    }
+
+    /**
+     * Create new cross-signing keys for the given key types. The public keys
+     * will be held in this class, while the private keys are passed off to the
+     * `saveCrossSigningKeys` application callback.
+     *
+     * @param {CrossSigningLevel} level The key types to reset
+     */
+    async resetKeys(level) {
+        if (!this._callbacks.saveCrossSigningKeys) {
+            throw new Error("No saveCrossSigningKeys callback supplied");
+        }
+
+        // If we're resetting the master key, we reset all keys
+        if (
+            level === undefined ||
+            level & CrossSigningLevel.MASTER ||
+            !this.keys.master
+        ) {
+            level = (
+                CrossSigningLevel.MASTER |
+                CrossSigningLevel.USER_SIGNING |
+                CrossSigningLevel.SELF_SIGNING
+            );
+        } else if (level === 0) {
+            return;
+        }
+
+        const privateKeys = {};
+        const keys = {};
+        let masterSigning;
+        let masterPub;
+
+        try {
+            if (level & CrossSigningLevel.MASTER) {
+                masterSigning = new global.Olm.PkSigning();
+                privateKeys.master = masterSigning.generate_seed();
+                masterPub = masterSigning.init_with_seed(privateKeys.master);
+                keys.master = {
+                    user_id: this.userId,
+                    usage: ['master'],
+                    keys: {
+                        ['ed25519:' + masterPub]: masterPub,
+                    },
+                };
+            } else {
+                [masterPub, masterSigning] = await this.getCrossSigningKey("master");
+            }
+
+            if (level & CrossSigningLevel.SELF_SIGNING) {
+                const sskSigning = new global.Olm.PkSigning();
+                try {
+                    privateKeys.self_signing = sskSigning.generate_seed();
+                    const sskPub = sskSigning.init_with_seed(privateKeys.self_signing);
+                    keys.self_signing = {
+                        user_id: this.userId,
+                        usage: ['self_signing'],
+                        keys: {
+                            ['ed25519:' + sskPub]: sskPub,
+                        },
+                    };
+                    pkSign(keys.self_signing, masterSigning, this.userId, masterPub);
+                } finally {
+                    sskSigning.free();
+                }
+            }
+
+            if (level & CrossSigningLevel.USER_SIGNING) {
+                const uskSigning = new global.Olm.PkSigning();
+                try {
+                    privateKeys.user_signing = uskSigning.generate_seed();
+                    const uskPub = uskSigning.init_with_seed(privateKeys.user_signing);
+                    keys.user_signing = {
+                        user_id: this.userId,
+                        usage: ['user_signing'],
+                        keys: {
+                            ['ed25519:' + uskPub]: uskPub,
+                        },
+                    };
+                    pkSign(keys.user_signing, masterSigning, this.userId, masterPub);
+                } finally {
+                    uskSigning.free();
+                }
+            }
+
+            Object.assign(this.keys, keys);
+            this._callbacks.saveCrossSigningKeys(privateKeys);
+        } finally {
+            if (masterSigning) {
+                masterSigning.free();
+            }
+        }
+    }
+
+    /**
+     * unsets the keys, used when another session has reset the keys, to disable cross-signing
+     */
+    clearKeys() {
+        this.keys = {};
+    }
+
+    setKeys(keys) {
+        const signingKeys = {};
+        if (keys.master) {
+            if (keys.master.user_id !== this.userId) {
+                const error = "Mismatched user ID " + keys.master.user_id +
+                      " in master key from " + this.userId;
+                logger.error(error);
+                throw new Error(error);
+            }
+            if (!this.keys.master) {
+                // this is the first key we've seen, so first-use is true
+                this.firstUse = true;
+            } else if (publicKeyFromKeyInfo(keys.master) !== this.getId()) {
+                // this is a different key, so first-use is false
+                this.firstUse = false;
+            } // otherwise, same key, so no change
+            signingKeys.master = keys.master;
+        } else if (this.keys.master) {
+            signingKeys.master = this.keys.master;
+        } else {
+            throw new Error("Tried to set cross-signing keys without a master key");
+        }
+        const masterKey = publicKeyFromKeyInfo(signingKeys.master);
+
+        // verify signatures
+        if (keys.user_signing) {
+            if (keys.user_signing.user_id !== this.userId) {
+                const error = "Mismatched user ID " + keys.master.user_id +
+                      " in user_signing key from " + this.userId;
+                logger.error(error);
+                throw new Error(error);
+            }
+            try {
+                pkVerify(keys.user_signing, masterKey, this.userId);
+            } catch (e) {
+                logger.error("invalid signature on user-signing key");
+                // FIXME: what do we want to do here?
+                throw e;
+            }
+        }
+        if (keys.self_signing) {
+            if (keys.self_signing.user_id !== this.userId) {
+                const error = "Mismatched user ID " + keys.master.user_id +
+                      " in self_signing key from " + this.userId;
+                logger.error(error);
+                throw new Error(error);
+            }
+            try {
+                pkVerify(keys.self_signing, masterKey, this.userId);
+            } catch (e) {
+                logger.error("invalid signature on self-signing key");
+                // FIXME: what do we want to do here?
+                throw e;
+            }
+        }
+
+        // if everything checks out, then save the keys
+        if (keys.master) {
+            this.keys.master = keys.master;
+            // if the master key is set, then the old self-signing and
+            // user-signing keys are obsolete
+            this.keys.self_signing = null;
+            this.keys.user_signing = null;
+        }
+        if (keys.self_signing) {
+            this.keys.self_signing = keys.self_signing;
+        }
+        if (keys.user_signing) {
+            this.keys.user_signing = keys.user_signing;
+        }
+    }
+
+    updateCrossSigningVerifiedBefore(isCrossSigningVerified) {
+        // It is critical that this value latches forward from false to true but
+        // never back to false to avoid a downgrade attack.
+        if (!this.crossSigningVerifiedBefore && isCrossSigningVerified) {
+            this.crossSigningVerifiedBefore = true;
+        }
+    }
+
+    async signObject(data, type) {
+        if (!this.keys[type]) {
+            throw new Error(
+                "Attempted to sign with " + type + " key but no such key present",
+            );
+        }
+        const [pubkey, signing] = await this.getCrossSigningKey(type);
+        try {
+            pkSign(data, signing, this.userId, pubkey);
+            return data;
+        } finally {
+            signing.free();
+        }
+    }
+
+    async signUser(key) {
+        if (!this.keys.user_signing) {
+            logger.info("No user signing key: not signing user");
+            return;
+        }
+        return this.signObject(key.keys.master, "user_signing");
+    }
+
+    async signDevice(userId, device) {
+        if (userId !== this.userId) {
+            throw new Error(
+                `Trying to sign ${userId}'s device; can only sign our own device`,
+            );
+        }
+        if (!this.keys.self_signing) {
+            logger.info("No self signing key: not signing device");
+            return;
+        }
+        return this.signObject(
+            {
+                algorithms: device.algorithms,
+                keys: device.keys,
+                device_id: device.deviceId,
+                user_id: userId,
+            }, "self_signing",
+        );
+    }
+
+    /**
+     * Check whether a given user is trusted.
+     *
+     * @param {CrossSigningInfo} userCrossSigning Cross signing info for user
+     *
+     * @returns {UserTrustLevel}
+     */
+    checkUserTrust(userCrossSigning) {
+        // if we're checking our own key, then it's trusted if the master key
+        // and self-signing key match
+        if (this.userId === userCrossSigning.userId
+            && this.getId() && this.getId() === userCrossSigning.getId()
+            && this.getId("self_signing")
+            && this.getId("self_signing") === userCrossSigning.getId("self_signing")
+        ) {
+            return new UserTrustLevel(true, true, this.firstUse);
+        }
+
+        if (!this.keys.user_signing) {
+            // If there's no user signing key, they can't possibly be verified.
+            // They may be TOFU trusted though.
+            return new UserTrustLevel(false, false, userCrossSigning.firstUse);
+        }
+
+        let userTrusted;
+        const userMaster = userCrossSigning.keys.master;
+        const uskId = this.getId('user_signing');
+        try {
+            pkVerify(userMaster, uskId, this.userId);
+            userTrusted = true;
+        } catch (e) {
+            userTrusted = false;
+        }
+        return new UserTrustLevel(
+            userTrusted,
+            userCrossSigning.crossSigningVerifiedBefore,
+            userCrossSigning.firstUse,
+        );
+    }
+
+    /**
+     * Check whether a given device is trusted.
+     *
+     * @param {CrossSigningInfo} userCrossSigning Cross signing info for user
+     * @param {module:crypto/deviceinfo} device The device to check
+     * @param {bool} localTrust Whether the device is trusted locally
+     * @param {bool} trustCrossSignedDevices Whether we trust cross signed devices
+     *
+     * @returns {DeviceTrustLevel}
+     */
+    checkDeviceTrust(userCrossSigning, device, localTrust, trustCrossSignedDevices) {
+        const userTrust = this.checkUserTrust(userCrossSigning);
+
+        const userSSK = userCrossSigning.keys.self_signing;
+        if (!userSSK) {
+            // if the user has no self-signing key then we cannot make any
+            // trust assertions about this device from cross-signing
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
+        }
+
+        const deviceObj = deviceToObject(device, userCrossSigning.userId);
+        try {
+            // if we can verify the user's SSK from their master key...
+            pkVerify(userSSK, userCrossSigning.getId(), userCrossSigning.userId);
+            // ...and this device's key from their SSK...
+            pkVerify(
+                deviceObj, publicKeyFromKeyInfo(userSSK), userCrossSigning.userId,
+            );
+            // ...then we trust this device as much as far as we trust the user
+            return DeviceTrustLevel.fromUserTrustLevel(
+                userTrust, localTrust, trustCrossSignedDevices,
+            );
+        } catch (e) {
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
+        }
+    }
+
+    /**
+     * @returns {object} Cache callbacks
+     */
+    getCacheCallbacks() {
+        return this._cacheCallbacks;
+    }
+}
+
+function deviceToObject(device, userId) {
+    return {
+        algorithms: device.algorithms,
+        keys: device.keys,
+        device_id: device.deviceId,
+        user_id: userId,
+        signatures: device.signatures,
+    };
+}
+
+export const CrossSigningLevel = {
+    MASTER: 4,
+    USER_SIGNING: 2,
+    SELF_SIGNING: 1,
+};
+
+/**
+ * Represents the ways in which we trust a user
+ */
+export class UserTrustLevel {
+    constructor(crossSigningVerified, crossSigningVerifiedBefore, tofu) {
+        this._crossSigningVerified = crossSigningVerified;
+        this._crossSigningVerifiedBefore = crossSigningVerifiedBefore;
+        this._tofu = tofu;
+    }
+
+    /**
+     * @returns {bool} true if this user is verified via any means
+     */
+    isVerified() {
+        return this.isCrossSigningVerified();
+    }
+
+    /**
+     * @returns {bool} true if this user is verified via cross signing
+     */
+    isCrossSigningVerified() {
+        return this._crossSigningVerified;
+    }
+
+    /**
+     * @returns {bool} true if we ever verified this user before (at least for
+     * the history of verifications observed by this device).
+     */
+    wasCrossSigningVerified() {
+        return this._crossSigningVerifiedBefore;
+    }
+
+    /**
+     * @returns {bool} true if this user's key is trusted on first use
+     */
+    isTofu() {
+        return this._tofu;
+    }
+}
+
+/**
+ * Represents the ways in which we trust a device
+ */
+export class DeviceTrustLevel {
+    constructor(crossSigningVerified, tofu, localVerified, trustCrossSignedDevices) {
+        this._crossSigningVerified = crossSigningVerified;
+        this._tofu = tofu;
+        this._localVerified = localVerified;
+        this._trustCrossSignedDevices = trustCrossSignedDevices;
+    }
+
+    static fromUserTrustLevel(userTrustLevel, localVerified, trustCrossSignedDevices) {
+        return new DeviceTrustLevel(
+            userTrustLevel._crossSigningVerified,
+            userTrustLevel._tofu,
+            localVerified,
+            trustCrossSignedDevices,
+        );
+    }
+
+    /**
+     * @returns {bool} true if this device is verified via any means
+     */
+    isVerified() {
+        return Boolean(this.isLocallyVerified() || (
+            this._trustCrossSignedDevices && this.isCrossSigningVerified()
+        ));
+    }
+
+    /**
+     * @returns {bool} true if this device is verified via cross signing
+     */
+    isCrossSigningVerified() {
+        return this._crossSigningVerified;
+    }
+
+    /**
+     * @returns {bool} true if this device is verified locally
+     */
+    isLocallyVerified() {
+        return this._localVerified;
+    }
+
+    /**
+     * @returns {bool} true if this device is trusted from a user's key
+     * that is trusted on first use
+     */
+    isTofu() {
+        return this._tofu;
+    }
+}
+
+export function createCryptoStoreCacheCallbacks(store, olmdevice) {
+    return {
+        getCrossSigningKeyCache: async function(type, _expectedPublicKey) {
+            const key = await new Promise((resolve) => {
+                return store.doTxn(
+                    'readonly',
+                    [IndexedDBCryptoStore.STORE_ACCOUNT],
+                    (txn) => {
+                        store.getSecretStorePrivateKey(txn, resolve, type);
+                    },
+                );
+            });
+
+            if (key && key.ciphertext) {
+                const pickleKey = Buffer.from(olmdevice._pickleKey);
+                const decrypted = await decryptAES(key, pickleKey, type);
+                return decodeBase64(decrypted);
+            } else {
+                return key;
+            }
+        },
+        storeCrossSigningKeyCache: async function(type, key) {
+            if (!(key instanceof Uint8Array)) {
+                throw new Error(
+                    `storeCrossSigningKeyCache expects Uint8Array, got ${key}`,
+                );
+            }
+            const pickleKey = Buffer.from(olmdevice._pickleKey);
+            key = await encryptAES(encodeBase64(key), pickleKey, type);
+            return store.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    store.storeSecretStorePrivateKey(txn, type, key);
+                },
+            );
+        },
+    };
+}
+
+/**
+ * Request cross-signing keys from another device during verification.
+ *
+ * @param {module:base-apis~MatrixBaseApis} baseApis base Matrix API interface
+ * @param {string} userId The user ID being verified
+ * @param {string} deviceId The device ID being verified
+ */
+export async function requestKeysDuringVerification(baseApis, userId, deviceId) {
+    // If this is a self-verification, ask the other party for keys
+    if (baseApis.getUserId() !== userId) {
+        return;
+    }
+    logger.log("Cross-signing: Self-verification done; requesting keys");
+    // This happens asynchronously, and we're not concerned about waiting for
+    // it.  We return here in order to test.
+    return new Promise((resolve, reject) => {
+        const client = baseApis;
+        const original = client._crypto._crossSigningInfo;
+
+        // We already have all of the infrastructure we need to validate and
+        // cache cross-signing keys, so instead of replicating that, here we set
+        // up callbacks that request them from the other device and call
+        // CrossSigningInfo.getCrossSigningKey() to validate/cache
+        const crossSigning = new CrossSigningInfo(
+            original.userId,
+            { getCrossSigningKey: async (type) => {
+                logger.debug("Cross-signing: requesting secret",
+                                type, deviceId);
+                const { promise } = client.requestSecret(
+                    `m.cross_signing.${type}`, [deviceId],
+                );
+                const result = await promise;
+                const decoded = decodeBase64(result);
+                return Uint8Array.from(decoded);
+            } },
+            original._cacheCallbacks,
+        );
+        crossSigning.keys = original.keys;
+
+        // XXX: get all keys out if we get one key out
+        // https://github.com/vector-im/element-web/issues/12604
+        // then change here to reject on the timeout
+        // Requests can be ignored, so don't wait around forever
+        const timeout = new Promise((resolve, reject) => {
+            setTimeout(
+                resolve,
+                KEY_REQUEST_TIMEOUT_MS,
+                new Error("Timeout"),
+            );
+        });
+
+        // also request and cache the key backup key
+        const backupKeyPromise = new Promise(async resolve => {
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            if (!cachedKey) {
+                logger.info("No cached backup key found. Requesting...");
+                const secretReq = client.requestSecret(
+                    'm.megolm_backup.v1', [deviceId],
+                );
+                const base64Key = await secretReq.promise;
+                logger.info("Got key backup key, decoding...");
+                const decodedKey = decodeBase64(base64Key);
+                logger.info("Decoded backup key, storing...");
+                client._crypto.storeSessionBackupPrivateKey(
+                    Uint8Array.from(decodedKey),
+                );
+                logger.info("Backup key stored. Starting backup restore...");
+                const backupInfo = await client.getKeyBackupVersion();
+                // no need to await for this - just let it go in the bg
+                client.restoreKeyBackupWithCache(
+                    undefined, undefined, backupInfo,
+                ).then(() => {
+                    logger.info("Backup restored.");
+                });
+            }
+            resolve();
+        });
+
+        // We call getCrossSigningKey() for its side-effects
+        return Promise.race([
+            Promise.all([
+                crossSigning.getCrossSigningKey("master"),
+                crossSigning.getCrossSigningKey("self_signing"),
+                crossSigning.getCrossSigningKey("user_signing"),
+                backupKeyPromise,
+            ]),
+            timeout,
+        ]).then(resolve, reject);
+    }).catch((e) => {
+        logger.warn("Cross-signing: failure while requesting keys:", e);
+    });
+}

src/crypto/DeviceList.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2017 Vector Creations Ltd
 Copyright 2018, 2019 New Vector Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,7 +15,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /**
  * @module crypto/DeviceList
@@ -22,12 +22,13 @@ limitations under the License.
  * Manages the list of other users' devices
  */
 
-import Promise from 'bluebird';
-
-import logger from '../logger';
-import DeviceInfo from './deviceinfo';
-import olmlib from './olmlib';
-import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+import {EventEmitter} from 'events';
+import {logger} from '../logger';
+import {DeviceInfo} from './deviceinfo';
+import {CrossSigningInfo} from './CrossSigning';
+import * as olmlib from './olmlib';
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import {chunkPromises, defer, sleep} from '../utils';
 
 
 /* State transition diagram for DeviceList._deviceTrackingStatus
@@ -60,8 +61,10 @@ const TRACKING_STATUS_UP_TO_DATE = 3;
 /**
  * @alias module:crypto/DeviceList
  */
-export default class DeviceList {
-    constructor(baseApis, cryptoStore, olmDevice) {
+export class DeviceList extends EventEmitter {
+    constructor(baseApis, cryptoStore, olmDevice, keyDownloadChunkSize = 250) {
+        super();
+
         this._cryptoStore = cryptoStore;
 
         // userId -> {
@@ -71,6 +74,11 @@ export default class DeviceList {
         // }
         this._devices = {};
 
+        // userId -> {
+        //     [key info]
+        // }
+        this._crossSigningInfo = {};
+
         // map of identity keys to the user who owns it
         this._userByIdentityKey = {};
 
@@ -90,6 +98,9 @@ export default class DeviceList {
         // userId -> promise
         this._keyDownloadsInProgressByUser = {};
 
+        // Maximum number of user IDs per request to prevent server overload (#1619)
+        this._keyDownloadChunkSize = keyDownloadChunkSize;
+
         // Set whenever changes are made other than setting the sync token
         this._dirty = false;
 
@@ -101,6 +112,9 @@ export default class DeviceList {
         this._savePromiseTime = null;
         // The timer used to delay the save
         this._saveTimer = null;
+        // True if we have fetched data from the server or loaded a non-empty
+        // set of device data from the store
+        this._hasFetched = null;
     }
 
     /**
@@ -110,7 +124,10 @@ export default class DeviceList {
         await this._cryptoStore.doTxn(
             'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                 this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    this._hasFetched = Boolean(deviceData && deviceData.devices);
                     this._devices = deviceData ? deviceData.devices : {},
+                    this._crossSigningInfo = deviceData ?
+                        deviceData.crossSigningInfo || {} : {};
                     this._deviceTrackingStatus = deviceData ?
                         deviceData.trackingStatus : {};
                     this._syncToken = deviceData ? deviceData.syncToken : null;
@@ -187,26 +204,33 @@ export default class DeviceList {
             const resolveSavePromise = this._resolveSavePromise;
             this._savePromiseTime = targetTime;
             this._saveTimer = setTimeout(() => {
-                logger.log('Saving device tracking data at token ' + this._syncToken);
+                logger.log('Saving device tracking data', this._syncToken);
+
                 // null out savePromise now (after the delay but before the write),
                 // otherwise we could return the existing promise when the save has
-                // actually already happened. Likewise for the dirty flag.
+                // actually already happened.
                 this._savePromiseTime = null;
                 this._saveTimer = null;
                 this._savePromise = null;
                 this._resolveSavePromise = null;
 
-                this._dirty = false;
                 this._cryptoStore.doTxn(
                     'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                         this._cryptoStore.storeEndToEndDeviceData({
                             devices: this._devices,
+                            crossSigningInfo: this._crossSigningInfo,
                             trackingStatus: this._deviceTrackingStatus,
                             syncToken: this._syncToken,
                         }, txn);
                     },
                 ).then(() => {
+                    // The device list is considered dirty until the write
+                    // completes.
+                    this._dirty = false;
                     resolveSavePromise();
+                }, err => {
+                    logger.error('Failed to save device tracking data', this._syncToken);
+                    logger.error(err);
                 });
             }, delay);
         }
@@ -300,6 +324,15 @@ export default class DeviceList {
         return stored;
     }
 
+    /**
+     * Returns a list of all user IDs the DeviceList knows about
+     *
+     * @return {array} All known user IDs
+     */
+    getKnownUserIds() {
+        return Object.keys(this._devices);
+    }
+
     /**
      * Get the stored device keys for a user id
      *
@@ -334,6 +367,17 @@ export default class DeviceList {
         return this._devices[userId];
     }
 
+    getStoredCrossSigningForUser(userId) {
+        if (!this._crossSigningInfo[userId]) return null;
+
+        return CrossSigningInfo.fromStorage(this._crossSigningInfo[userId], userId);
+    }
+
+    storeCrossSigningForUser(userId, info) {
+        this._crossSigningInfo[userId] = info;
+        this._dirty = true;
+    }
+
     /**
      * Get the stored keys for a single device
      *
@@ -351,6 +395,26 @@ export default class DeviceList {
         return DeviceInfo.fromStorage(devs[deviceId], deviceId);
     }
 
+    /**
+     * Get a user ID by one of their device's curve25519 identity key
+     *
+     * @param {string} algorithm  encryption algorithm
+     * @param {string} senderKey  curve25519 key to match
+     *
+     * @return {string} user ID
+     */
+    getUserByIdentityKey(algorithm, senderKey) {
+        if (
+            algorithm !== olmlib.OLM_ALGORITHM &&
+            algorithm !== olmlib.MEGOLM_ALGORITHM
+        ) {
+            // we only deal in olm keys
+            return null;
+        }
+
+        return this._userByIdentityKey[senderKey];
+    }
+
     /**
      * Find a device by curve25519 identity key
      *
@@ -360,19 +424,11 @@ export default class DeviceList {
      * @return {module:crypto/deviceinfo?}
      */
     getDeviceByIdentityKey(algorithm, senderKey) {
-        const userId = this._userByIdentityKey[senderKey];
+        const userId = this.getUserByIdentityKey(algorithm, senderKey);
         if (!userId) {
             return null;
         }
 
-        if (
-            algorithm !== olmlib.OLM_ALGORITHM &&
-            algorithm !== olmlib.MEGOLM_ALGORITHM
-        ) {
-            // we only deal in olm keys
-            return null;
-        }
-
         const devices = this._devices[userId];
         if (!devices) {
             return null;
@@ -561,6 +617,10 @@ export default class DeviceList {
         }
     }
 
+    setRawStoredCrossSigningForUser(userId, info) {
+        this._crossSigningInfo[userId] = info;
+    }
+
     /**
      * Fire off download update requests for the given users, and update the
      * device list tracking status for them, and the
@@ -568,7 +628,7 @@ export default class DeviceList {
      *
      * @param {String[]} users  list of userIds
      *
-     * @return {module:client.Promise} resolves when all the users listed have
+     * @return {Promise} resolves when all the users listed have
      *     been updated. rejects if there was a problem updating any of the
      *     users.
      */
@@ -599,6 +659,7 @@ export default class DeviceList {
         });
 
         const finished = (success) => {
+            this.emit("crypto.willUpdateDevices", users, !this._hasFetched);
             users.forEach((u) => {
                 this._dirty = true;
 
@@ -624,6 +685,8 @@ export default class DeviceList {
                 }
             });
             this.saveIfDirty();
+            this.emit("crypto.devicesUpdated", users, !this._hasFetched);
+            this._hasFetched = true;
         };
 
         return prom;
@@ -672,7 +735,7 @@ class DeviceListUpdateSerialiser {
      * @param {String} syncToken sync token to pass in the query request, to
      *     help the HS give the most recent results
      *
-     * @return {module:client.Promise} resolves when all the users listed have
+     * @return {Promise} resolves when all the users listed have
      *     been updated. rejects if there was a problem updating any of the
      *     users.
      */
@@ -682,7 +745,7 @@ class DeviceListUpdateSerialiser {
         });
 
         if (!this._queuedQueryDeferred) {
-            this._queuedQueryDeferred = Promise.defer();
+            this._queuedQueryDeferred = defer();
         }
 
         // We always take the new sync token and just use the latest one we've
@@ -720,25 +783,41 @@ class DeviceListUpdateSerialiser {
             opts.token = this._syncToken;
         }
 
-        this._baseApis.downloadKeysForUsers(
-            downloadUsers, opts,
-        ).then((res) => {
-            const dk = res.device_keys || {};
+        const factories = [];
+        for (let i = 0; i < downloadUsers.length; i += this._deviceList._keyDownloadChunkSize) {
+            const userSlice = downloadUsers.slice(i, i + this._deviceList._keyDownloadChunkSize);
+            factories.push(() => this._baseApis.downloadKeysForUsers(userSlice, opts));
+        }
 
-            // do each user in a separate promise, to avoid wedging the CPU
-            // (https://github.com/vector-im/riot-web/issues/3158)
+        chunkPromises(factories, 3).then(async (responses) => {
+            const dk = Object.assign({}, ...(responses.map(res => res.device_keys || {})));
+            const masterKeys = Object.assign({}, ...(responses.map(res => res.master_keys || {})));
+            const ssks = Object.assign({}, ...(responses.map(res => res.self_signing_keys || {})));
+            const usks = Object.assign({}, ...(responses.map(res => res.user_signing_keys || {})));
+
+            // yield to other things that want to execute in between users, to
+            // avoid wedging the CPU
+            // (https://github.com/vector-im/element-web/issues/3158)
             //
             // of course we ought to do this in a web worker or similar, but
             // this serves as an easy solution for now.
-            let prom = Promise.resolve();
             for (const userId of downloadUsers) {
-                prom = prom.delay(5).then(() => {
-                    return this._processQueryResponseForUser(userId, dk[userId]);
-                });
+                await sleep(5);
+                try {
+                    await this._processQueryResponseForUser(
+                        userId, dk[userId], {
+                            master: masterKeys[userId],
+                            self_signing: ssks[userId],
+                            user_signing: usks[userId],
+                        },
+                    );
+                } catch (e) {
+                    // log the error but continue, so that one bad key
+                    // doesn't kill the whole process
+                    logger.error(`Error processing keys for ${userId}:`, e);
+                }
             }
-
-            return prom;
-        }).done(() => {
+        }).then(() => {
             logger.log('Completed key download for ' + downloadUsers);
 
             this._downloadInProgress = false;
@@ -757,36 +836,66 @@ class DeviceListUpdateSerialiser {
         return deferred.promise;
     }
 
-    async _processQueryResponseForUser(userId, response) {
-        logger.log('got keys for ' + userId + ':', response);
+    async _processQueryResponseForUser(
+        userId, dkResponse, crossSigningResponse,
+    ) {
+        logger.log('got device keys for ' + userId + ':', dkResponse);
+        logger.log('got cross-signing keys for ' + userId + ':', crossSigningResponse);
 
-        // map from deviceid -> deviceinfo for this user
-        const userStore = {};
-        const devs = this._deviceList.getRawStoredDevicesForUser(userId);
-        if (devs) {
-            Object.keys(devs).forEach((deviceId) => {
-                const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
-                userStore[deviceId] = d;
+        {
+            // map from deviceid -> deviceinfo for this user
+            const userStore = {};
+            const devs = this._deviceList.getRawStoredDevicesForUser(userId);
+            if (devs) {
+                Object.keys(devs).forEach((deviceId) => {
+                    const d = DeviceInfo.fromStorage(devs[deviceId], deviceId);
+                    userStore[deviceId] = d;
+                });
+            }
+
+            await _updateStoredDeviceKeysForUser(
+                this._olmDevice, userId, userStore, dkResponse || {},
+                this._baseApis.getUserId(), this._baseApis.deviceId,
+            );
+
+            // put the updates into the object that will be returned as our results
+            const storage = {};
+            Object.keys(userStore).forEach((deviceId) => {
+                storage[deviceId] = userStore[deviceId].toStorage();
             });
+
+            this._deviceList._setRawStoredDevicesForUser(userId, storage);
         }
 
-        await _updateStoredDeviceKeysForUser(
-            this._olmDevice, userId, userStore, response || {},
-        );
+        // now do the same for the cross-signing keys
+        {
+            // FIXME: should we be ignoring empty cross-signing responses, or
+            // should we be dropping the keys?
+            if (crossSigningResponse
+                && (crossSigningResponse.master || crossSigningResponse.self_signing
+                    || crossSigningResponse.user_signing)) {
+                const crossSigning
+                      = this._deviceList.getStoredCrossSigningForUser(userId)
+                      || new CrossSigningInfo(userId);
 
-        // put the updates into thr object that will be returned as our results
-        const storage = {};
-        Object.keys(userStore).forEach((deviceId) => {
-            storage[deviceId] = userStore[deviceId].toStorage();
-        });
+                crossSigning.setKeys(crossSigningResponse);
 
-        this._deviceList._setRawStoredDevicesForUser(userId, storage);
+                this._deviceList.setRawStoredCrossSigningForUser(
+                    userId, crossSigning.toStorage(),
+                );
+
+                // NB. Unlike most events in the js-sdk, this one is internal to the
+                // js-sdk and is not re-emitted
+                this._deviceList.emit('userCrossSigningUpdated', userId);
+            }
+        }
     }
 }
 
 
-async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
-        userResult) {
+async function _updateStoredDeviceKeysForUser(
+    _olmDevice, userId, userStore, userResult, localUserId, localDeviceId,
+) {
     let updated = false;
 
     // remove any devices in the store which aren't in the response
@@ -796,6 +905,13 @@ async function _updateStoredDeviceKeysForUser(_olmDevice, userId, userStore,
         }
 
         if (!(deviceId in userResult)) {
+            if (userId === localUserId && deviceId === localDeviceId) {
+                logger.warn(
+                    `Local device ${deviceId} missing from sync, skipping removal`,
+                );
+                continue;
+            }
+
             logger.log("Device " + userId + ":" + deviceId +
                 " has been removed");
             delete userStore[deviceId];
@@ -854,6 +970,7 @@ async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
     }
 
     const unsigned = deviceResult.unsigned || {};
+    const signatures = deviceResult.signatures || {};
 
     try {
         await olmlib.verifySignature(_olmDevice, deviceResult, userId, deviceId, signKey);
@@ -886,5 +1003,6 @@ async function _storeDeviceKeys(_olmDevice, userStore, deviceResult) {
     deviceStore.keys = deviceResult.keys || {};
     deviceStore.algorithms = deviceResult.algorithms || [];
     deviceStore.unsigned = unsigned;
+    deviceStore.signatures = signatures;
     return true;
 }

src/crypto/EncryptionSetup.js

@@ -0,0 +1,357 @@
+import { logger } from "../logger";
+import {MatrixEvent} from "../models/event";
+import {EventEmitter} from "events";
+import {createCryptoStoreCacheCallbacks} from "./CrossSigning";
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import {
+    PREFIX_UNSTABLE,
+} from "../http-api";
+
+/**
+ * Builds an EncryptionSetupOperation by calling any of the add.. methods.
+ * Once done, `buildOperation()` can be called which allows to apply to operation.
+ *
+ * This is used as a helper by Crypto to keep track of all the network requests
+ * and other side-effects of bootstrapping, so it can be applied in one go (and retried in the future)
+ * Also keeps track of all the private keys created during bootstrapping, so we don't need to prompt for them
+ * more than once.
+ */
+export class EncryptionSetupBuilder {
+    /**
+     * @param {Object.<String, MatrixEvent>} accountData pre-existing account data, will only be read, not written.
+     * @param {CryptoCallbacks} delegateCryptoCallbacks crypto callbacks to delegate to if the key isn't in cache yet
+     */
+    constructor(accountData, delegateCryptoCallbacks) {
+        this.accountDataClientAdapter = new AccountDataClientAdapter(accountData);
+        this.crossSigningCallbacks = new CrossSigningCallbacks();
+        this.ssssCryptoCallbacks = new SSSSCryptoCallbacks(delegateCryptoCallbacks);
+
+        this._crossSigningKeys = null;
+        this._keySignatures = null;
+        this._keyBackupInfo = null;
+    }
+
+    /**
+     * Adds new cross-signing public keys
+     *
+     * @param {function} authUpload Function called to await an interactive auth
+     * flow when uploading device signing keys.
+     * Args:
+     *     {function} A function that makes the request requiring auth. Receives
+     *     the auth data as an object. Can be called multiple times, first with
+     *     an empty authDict, to obtain the flows.
+     * @param {Object} keys the new keys
+     */
+    addCrossSigningKeys(authUpload, keys) {
+        this._crossSigningKeys = {authUpload, keys};
+    }
+
+    /**
+     * Adds the key backup info to be updated on the server
+     *
+     * Used either to create a new key backup, or add signatures
+     * from the new MSK.
+     *
+     * @param {Object} keyBackupInfo as received from/sent to the server
+     */
+    addSessionBackup(keyBackupInfo) {
+        this._keyBackupInfo = keyBackupInfo;
+    }
+
+    /**
+     * Adds the session backup private key to be updated in the local cache
+     *
+     * Used after fixing the format of the key
+     *
+     * @param {Uint8Array} privateKey
+     */
+    addSessionBackupPrivateKeyToCache(privateKey) {
+        this._sessionBackupPrivateKey = privateKey;
+    }
+
+    /**
+     * Add signatures from a given user and device/x-sign key
+     * Used to sign the new cross-signing key with the device key
+     *
+     * @param {String} userId
+     * @param {String} deviceId
+     * @param {String} signature
+     */
+    addKeySignature(userId, deviceId, signature) {
+        if (!this._keySignatures) {
+            this._keySignatures = {};
+        }
+        const userSignatures = this._keySignatures[userId] || {};
+        this._keySignatures[userId] = userSignatures;
+        userSignatures[deviceId] = signature;
+    }
+
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        return this.accountDataClientAdapter.setAccountData(type, content);
+    }
+
+    /**
+     * builds the operation containing all the parts that have been added to the builder
+     * @return {EncryptionSetupOperation}
+     */
+    buildOperation() {
+        const accountData = this.accountDataClientAdapter._values;
+        return new EncryptionSetupOperation(
+            accountData,
+            this._crossSigningKeys,
+            this._keyBackupInfo,
+            this._keySignatures,
+        );
+    }
+
+    /**
+     * Stores the created keys locally.
+     *
+     * This does not yet store the operation in a way that it can be restored,
+     * but that is the idea in the future.
+     *
+     * @param  {Crypto} crypto
+     * @return {Promise}
+     */
+    async persist(crypto) {
+        // store private keys in cache
+        if (this._crossSigningKeys) {
+            const cacheCallbacks = createCryptoStoreCacheCallbacks(
+                crypto._cryptoStore, crypto._olmDevice);
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                logger.log(`Cache ${type} cross-signing private key locally`);
+                const privateKey = this.crossSigningCallbacks.privateKeys.get(type);
+                await cacheCallbacks.storeCrossSigningKeyCache(type, privateKey);
+            }
+            // store own cross-sign pubkeys as trusted
+            await crypto._cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    crypto._cryptoStore.storeCrossSigningKeys(
+                        txn, this._crossSigningKeys.keys);
+                },
+            );
+        }
+        // store session backup key in cache
+        if (this._sessionBackupPrivateKey) {
+            await crypto.storeSessionBackupPrivateKey(this._sessionBackupPrivateKey);
+        }
+    }
+}
+
+/**
+ * Can be created from EncryptionSetupBuilder, or
+ * (in a follow-up PR, not implemented yet) restored from storage, to retry.
+ *
+ * It does not have knowledge of any private keys, unlike the builder.
+ */
+export class EncryptionSetupOperation {
+    /**
+     * @param  {Map<String, Object>} accountData
+     * @param  {Object} crossSigningKeys
+     * @param  {Object} keyBackupInfo
+     * @param  {Object} keySignatures
+     */
+    constructor(accountData, crossSigningKeys, keyBackupInfo, keySignatures) {
+        this._accountData = accountData;
+        this._crossSigningKeys = crossSigningKeys;
+        this._keyBackupInfo = keyBackupInfo;
+        this._keySignatures = keySignatures;
+    }
+
+    /**
+     * Runs the (remaining part of, in the future) operation by sending requests to the server.
+     * @param  {Crypto} crypto
+     */
+    async apply(crypto) {
+        const baseApis = crypto._baseApis;
+        // upload cross-signing keys
+        if (this._crossSigningKeys) {
+            const keys = {};
+            for (const [name, key] of Object.entries(this._crossSigningKeys.keys)) {
+                keys[name + "_key"] = key;
+            }
+
+            // We must only call `uploadDeviceSigningKeys` from inside this auth
+            // helper to ensure we properly handle auth errors.
+            await this._crossSigningKeys.authUpload(authDict => {
+                return baseApis.uploadDeviceSigningKeys(authDict, keys);
+            });
+
+            // pass the new keys to the main instance of our own CrossSigningInfo.
+            crypto._crossSigningInfo.setKeys(this._crossSigningKeys.keys);
+        }
+        // set account data
+        if (this._accountData) {
+            for (const [type, content] of this._accountData) {
+                await baseApis.setAccountData(type, content);
+            }
+        }
+        // upload first cross-signing signatures with the new key
+        // (e.g. signing our own device)
+        if (this._keySignatures) {
+            await baseApis.uploadKeySignatures(this._keySignatures);
+        }
+        // need to create/update key backup info
+        if (this._keyBackupInfo) {
+            if (this._keyBackupInfo.version) {
+                // session backup signature
+                // The backup is trusted because the user provided the private key.
+                // Sign the backup with the cross signing key so the key backup can
+                // be trusted via cross-signing.
+                await baseApis._http.authedRequest(
+                    undefined, "PUT", "/room_keys/version/" + this._keyBackupInfo.version,
+                    undefined, {
+                        algorithm: this._keyBackupInfo.algorithm,
+                        auth_data: this._keyBackupInfo.auth_data,
+                    },
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            } else {
+                // add new key backup
+                await baseApis._http.authedRequest(
+                    undefined, "POST", "/room_keys/version",
+                    undefined, this._keyBackupInfo,
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            }
+        }
+    }
+}
+
+
+/**
+ * Catches account data set by SecretStorage during bootstrapping by
+ * implementing the methods related to account data in MatrixClient
+ */
+class AccountDataClientAdapter extends EventEmitter {
+    /**
+     * @param  {Object.<String, MatrixEvent>} accountData existing account data
+     */
+    constructor(accountData) {
+        super();
+        this._existingValues = accountData;
+        this._values = new Map();
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Promise<Object>} the content of the account data
+     */
+    getAccountDataFromServer(type) {
+        return Promise.resolve(this.getAccountData(type));
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Object} the content of the account data
+     */
+    getAccountData(type) {
+        const modifiedValue = this._values.get(type);
+        if (modifiedValue) {
+            return modifiedValue;
+        }
+        const existingValue = this._existingValues[type];
+        if (existingValue) {
+            return existingValue.getContent();
+        }
+        return null;
+    }
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        const lastEvent = this._values.get(type);
+        this._values.set(type, content);
+        // ensure accountData is emitted on the next tick,
+        // as SecretStorage listens for it while calling this method
+        // and it seems to rely on this.
+        return Promise.resolve().then(() => {
+            const event = new MatrixEvent({type, content});
+            this.emit("accountData", event, lastEvent);
+        });
+    }
+}
+
+/**
+ * Catches the private cross-signing keys set during bootstrapping
+ * by both cache callbacks (see createCryptoStoreCacheCallbacks) as non-cache callbacks.
+ * See CrossSigningInfo constructor
+ */
+class CrossSigningCallbacks {
+    constructor() {
+        this.privateKeys = new Map();
+    }
+
+    // cache callbacks
+    getCrossSigningKeyCache(type, expectedPublicKey) {
+        return this.getCrossSigningKey(type, expectedPublicKey);
+    }
+
+    storeCrossSigningKeyCache(type, key) {
+        this.privateKeys.set(type, key);
+        return Promise.resolve();
+    }
+
+    // non-cache callbacks
+    getCrossSigningKey(type, _expectedPubkey) {
+        return Promise.resolve(this.privateKeys.get(type));
+    }
+
+    saveCrossSigningKeys(privateKeys) {
+        for (const [type, privateKey] of Object.entries(privateKeys)) {
+            this.privateKeys.set(type, privateKey);
+        }
+    }
+}
+
+/**
+ * Catches the 4S private key set during bootstrapping by implementing
+ * the SecretStorage crypto callbacks
+ */
+class SSSSCryptoCallbacks {
+    constructor(delegateCryptoCallbacks) {
+        this._privateKeys = new Map();
+        this._delegateCryptoCallbacks = delegateCryptoCallbacks;
+    }
+
+    async getSecretStorageKey({ keys }, name) {
+        for (const keyId of Object.keys(keys)) {
+            const privateKey = this._privateKeys.get(keyId);
+            if (privateKey) {
+                return [keyId, privateKey];
+            }
+        }
+        // if we don't have the key cached yet, ask
+        // for it to the general crypto callbacks and cache it
+        if (this._delegateCryptoCallbacks) {
+            const result = await this._delegateCryptoCallbacks.
+                getSecretStorageKey({keys}, name);
+            if (result) {
+                const [keyId, privateKey] = result;
+                this._privateKeys.set(keyId, privateKey);
+            }
+            return result;
+        }
+    }
+
+    addPrivateKey(keyId, keyInfo, privKey) {
+        this._privateKeys.set(keyId, privKey);
+        // Also pass along to application to cache if it wishes
+        if (
+            this._delegateCryptoCallbacks &&
+            this._delegateCryptoCallbacks.cacheSecretStorageKey
+        ) {
+            this._delegateCryptoCallbacks.cacheSecretStorageKey(keyId, keyInfo, privKey);
+        }
+    }
+}

src/crypto/OlmDevice.js

@@ -1,6 +1,7 @@
 /*
 Copyright 2016 OpenMarket Ltd
 Copyright 2017, 2019 New Vector Ltd
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,8 +16,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import logger from '../logger';
-import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+import {logger} from '../logger';
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import * as algorithms from './algorithms';
 
 // The maximum size of an event is 65K, and we base64 the content, so this is a
 // reasonable approximation to the biggest plaintext we can encrypt.
@@ -34,9 +36,15 @@ function checkPayloadLength(payloadString) {
         // Note that even if we manage to do the encryption, the message send may fail,
         // because by the time we've wrapped the ciphertext in the event object, it may
         // exceed 65K. But at least we won't just fail with "abort()" in that case.
-        throw new Error("Message too long (" + payloadString.length + " bytes). " +
+        const err = new Error("Message too long (" + payloadString.length + " bytes). " +
                         "The maximum for an encrypted message is " +
                         MAX_PLAINTEXT_LENGTH + " bytes.");
+        // TODO: [TypeScript] We should have our own error types
+        err.data = {
+            errcode: "M_TOO_LARGE",
+            error: "Payload too large for encrypted message",
+        };
+        throw err;
     }
 }
 
@@ -69,7 +77,7 @@ function checkPayloadLength(payloadString) {
  * @property {string} deviceCurve25519Key   Curve25519 key for the account
  * @property {string} deviceEd25519Key      Ed25519 key for the account
  */
-function OlmDevice(cryptoStore) {
+export function OlmDevice(cryptoStore) {
     this._cryptoStore = cryptoStore;
     this._pickleKey = "DEFAULT_KEY";
 
@@ -103,22 +111,61 @@ function OlmDevice(cryptoStore) {
     // Keep track of sessions that we're starting, so that we don't start
     // multiple sessions for the same device at the same time.
     this._sessionsInProgress = {};
+
+    // Used by olm to serialise prekey message decryptions
+    this._olmPrekeyPromise = Promise.resolve();
 }
 
 /**
  * Initialise the OlmAccount. This must be called before any other operations
  * on the OlmDevice.
  *
+ * Data from an exported Olm device can be provided
+ * in order to re-create this device.
+ *
  * Attempts to load the OlmAccount from the crypto store, or creates one if none is
  * found.
  *
  * Reads the device keys from the OlmAccount object.
+ *
+ * @param {object} opts
+ * @param {object} opts.fromExportedDevice (Optional) data from exported device
+ *     that must be re-created.
+ *     If present, opts.pickleKey is ignored
+ *     (exported data already provides a pickle key)
+ * @param {object} opts.pickleKey (Optional) pickle key to set instead of default one
  */
-OlmDevice.prototype.init = async function() {
+OlmDevice.prototype.init = async function(opts = {}) {
     let e2eKeys;
     const account = new global.Olm.Account();
+
+    const { pickleKey, fromExportedDevice } = opts;
+
     try {
-        await _initialiseAccount(this._cryptoStore, this._pickleKey, account);
+        if (fromExportedDevice) {
+            if (pickleKey) {
+                logger.warn(
+                    'ignoring opts.pickleKey'
+                    + ' because opts.fromExportedDevice is present.',
+                );
+            }
+            this._pickleKey = fromExportedDevice.pickleKey;
+            await _initialiseFromExportedDevice(
+                fromExportedDevice,
+                this._cryptoStore,
+                this._pickleKey,
+                account,
+            );
+        } else {
+            if (pickleKey) {
+                this._pickleKey = pickleKey;
+            }
+            await _initialiseAccount(
+                this._cryptoStore,
+                this._pickleKey,
+                account,
+            );
+        }
         e2eKeys = JSON.parse(account.identity_keys());
 
         this._maxOneTimeKeys = account.max_number_of_one_time_keys();
@@ -130,18 +177,67 @@ OlmDevice.prototype.init = async function() {
     this.deviceEd25519Key = e2eKeys.ed25519;
 };
 
-async function _initialiseAccount(cryptoStore, pickleKey, account) {
-    await cryptoStore.doTxn('readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT], (txn) => {
-        cryptoStore.getAccount(txn, (pickledAccount) => {
-            if (pickledAccount !== null) {
-                account.unpickle(pickleKey, pickledAccount);
-            } else {
-                account.create();
-                pickledAccount = account.pickle(pickleKey);
-                cryptoStore.storeAccount(txn, pickledAccount);
-            }
-        });
+/**
+ * Populates the crypto store using data that was exported from an existing device.
+ * Note that for now only the “account” and “sessions” stores are populated;
+ * Other stores will be as with a new device.
+ *
+ * @param {Object} exportedData Data exported from another device
+ *     through the “export” method.
+ * @param {module:crypto/store/base~CryptoStore} cryptoStore storage for the crypto layer
+ * @param {string} pickleKey the key that was used to pickle the exported data
+ * @param {Olm.Account} account an olm account to initialize
+ */
+async function _initialiseFromExportedDevice(
+    exportedData,
+    cryptoStore,
+    pickleKey,
+    account,
+) {
+    await cryptoStore.doTxn(
+        'readwrite',
+        [
+            IndexedDBCryptoStore.STORE_ACCOUNT,
+            IndexedDBCryptoStore.STORE_SESSIONS,
+        ],
+        (txn) => {
+            cryptoStore.storeAccount(txn, exportedData.pickledAccount);
+            exportedData.sessions.forEach((session) => {
+                const {
+                    deviceKey,
+                    sessionId,
+                } = session;
+                const sessionInfo = {
+                    session: session.session,
+                    lastReceivedMessageTs: session.lastReceivedMessageTs,
+                };
+                cryptoStore.storeEndToEndSession(
+                    deviceKey,
+                    sessionId,
+                    sessionInfo,
+                    txn,
+                );
+            });
     });
+    account.unpickle(pickleKey, exportedData.pickledAccount);
+}
+
+async function _initialiseAccount(cryptoStore, pickleKey, account) {
+    await cryptoStore.doTxn(
+        'readwrite',
+        [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            cryptoStore.getAccount(txn, (pickledAccount) => {
+                if (pickledAccount !== null) {
+                    account.unpickle(pickleKey, pickledAccount);
+                } else {
+                    account.create();
+                    pickledAccount = account.pickle(pickleKey);
+                    cryptoStore.storeAccount(txn, pickledAccount);
+                }
+            });
+        },
+    );
 }
 
 /**
@@ -189,6 +285,38 @@ OlmDevice.prototype._storeAccount = function(txn, account) {
     this._cryptoStore.storeAccount(txn, account.pickle(this._pickleKey));
 };
 
+/**
+ * Export data for re-creating the Olm device later.
+ * TODO export data other than just account and (P2P) sessions.
+ *
+ * @return {Promise<object>} The exported data
+*/
+OlmDevice.prototype.export = async function() {
+    const result = {
+        pickleKey: this._pickleKey,
+    };
+    await this._cryptoStore.doTxn(
+        'readonly',
+        [
+            IndexedDBCryptoStore.STORE_ACCOUNT,
+            IndexedDBCryptoStore.STORE_SESSIONS,
+        ],
+        (txn) => {
+            this._cryptoStore.getAccount(txn, (pickledAccount) => {
+                result.pickledAccount = pickledAccount;
+            });
+            result.sessions = [];
+            // Note that the pickledSession object we get in the callback
+            // is not exactly the same thing you get in method _getSession
+            // see documentation of IndexedDBCryptoStore.getAllEndToEndSessions
+            this._cryptoStore.getAllEndToEndSessions(txn, (pickledSession) => {
+                result.sessions.push(pickledSession);
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * extract an OlmSession from the session store and call the given function
  * The session is useable only within the callback passed to this
@@ -349,6 +477,36 @@ OlmDevice.prototype.generateOneTimeKeys = function(numKeys) {
     );
 };
 
+/**
+ * Generate a new fallback keys
+ *
+ * @return {Promise} Resolved once the account is saved back having generated the key
+ */
+OlmDevice.prototype.generateFallbackKey = async function() {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                account.generate_fallback_key();
+                this._storeAccount(txn, account);
+            });
+        },
+    );
+};
+
+OlmDevice.prototype.getFallbackKey = async function() {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._getAccount(txn, (account) => {
+                result = JSON.parse(account.fallback_key());
+            });
+        },
+    );
+    return result;
+};
+
 /**
  * Generate a new outbound session
  *
@@ -387,6 +545,7 @@ OlmDevice.prototype.createOutboundSession = async function(
                 }
             });
         },
+        logger.withPrefix("[createOutboundSession]"),
     );
     return newSessionId;
 };
@@ -447,6 +606,7 @@ OlmDevice.prototype.createInboundSession = async function(
                 }
             });
         },
+        logger.withPrefix("[createInboundSession]"),
     );
 
     return result;
@@ -461,8 +621,10 @@ OlmDevice.prototype.createInboundSession = async function(
  * @return {Promise<string[]>}  a list of known session ids for the device
  */
 OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityKey) {
+    const log = logger.withPrefix("[getSessionIdsForDevice]");
+
     if (this._sessionsInProgress[theirDeviceIdentityKey]) {
-        logger.log("waiting for session to be created");
+        log.debug(`Waiting for Olm session for ${theirDeviceIdentityKey} to be created`);
         try {
             await this._sessionsInProgress[theirDeviceIdentityKey];
         } catch (e) {
@@ -480,6 +642,7 @@ OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityK
                 },
             );
         },
+        log,
     );
 
     return sessionIds;
@@ -493,13 +656,14 @@ OlmDevice.prototype.getSessionIdsForDevice = async function(theirDeviceIdentityK
  * @param {boolean} nowait Don't wait for an in-progress session to complete.
  *     This should only be set to true of the calling function is the function
  *     that marked the session as being in-progress.
+ * @param {Logger} [log] A possibly customised log
  * @return {Promise<?string>}  session id, or null if no established session
  */
 OlmDevice.prototype.getSessionIdForDevice = async function(
-    theirDeviceIdentityKey, nowait,
+    theirDeviceIdentityKey, nowait, log,
 ) {
     const sessionInfos = await this.getSessionInfoForDevice(
-        theirDeviceIdentityKey, nowait,
+        theirDeviceIdentityKey, nowait, log,
     );
 
     if (sessionInfos.length === 0) {
@@ -539,11 +703,16 @@ OlmDevice.prototype.getSessionIdForDevice = async function(
  * @param {boolean} nowait Don't wait for an in-progress session to complete.
  *     This should only be set to true of the calling function is the function
  *     that marked the session as being in-progress.
+ * @param {Logger} [log] A possibly customised log
  * @return {Array.<{sessionId: string, hasReceivedMessage: Boolean}>}
  */
-OlmDevice.prototype.getSessionInfoForDevice = async function(deviceIdentityKey, nowait) {
+OlmDevice.prototype.getSessionInfoForDevice = async function(
+    deviceIdentityKey, nowait, log = logger,
+) {
+    log = log.withPrefix("[getSessionInfoForDevice]");
+
     if (this._sessionsInProgress[deviceIdentityKey] && !nowait) {
-        logger.log("waiting for session to be created");
+        log.debug(`Waiting for Olm session for ${deviceIdentityKey} to be created`);
         try {
             await this._sessionsInProgress[deviceIdentityKey];
         } catch (e) {
@@ -569,6 +738,7 @@ OlmDevice.prototype.getSessionInfoForDevice = async function(deviceIdentityKey,
                 }
             });
         },
+        log,
     );
 
     return info;
@@ -594,10 +764,16 @@ OlmDevice.prototype.encryptMessage = async function(
         'readwrite', [IndexedDBCryptoStore.STORE_SESSIONS],
         (txn) => {
             this._getSession(theirDeviceIdentityKey, sessionId, txn, (sessionInfo) => {
+                const sessionDesc = sessionInfo.session.describe();
+                logger.log(
+                    "encryptMessage: Olm Session ID " + sessionId + " to " +
+                    theirDeviceIdentityKey + ": " + sessionDesc,
+                );
                 res = sessionInfo.session.encrypt(payloadString);
                 this._saveSession(theirDeviceIdentityKey, sessionInfo, txn);
             });
         },
+        logger.withPrefix("[encryptMessage]"),
     );
     return res;
 };
@@ -621,11 +797,17 @@ OlmDevice.prototype.decryptMessage = async function(
         'readwrite', [IndexedDBCryptoStore.STORE_SESSIONS],
         (txn) => {
             this._getSession(theirDeviceIdentityKey, sessionId, txn, (sessionInfo) => {
+                const sessionDesc = sessionInfo.session.describe();
+                logger.log(
+                    "decryptMessage: Olm Session ID " + sessionId + " from " +
+                    theirDeviceIdentityKey + ": " + sessionDesc,
+                );
                 payloadString = sessionInfo.session.decrypt(messageType, ciphertext);
                 sessionInfo.lastReceivedMessageTs = Date.now();
                 this._saveSession(theirDeviceIdentityKey, sessionInfo, txn);
             });
         },
+        logger.withPrefix("[decryptMessage]"),
     );
     return payloadString;
 };
@@ -657,10 +839,23 @@ OlmDevice.prototype.matchesSession = async function(
                 matches = sessionInfo.session.matches_inbound(ciphertext);
             });
         },
+        logger.withPrefix("[matchesSession]"),
     );
     return matches;
 };
 
+OlmDevice.prototype.recordSessionProblem = async function(deviceKey, type, fixed) {
+    await this._cryptoStore.storeEndToEndSessionProblem(deviceKey, type, fixed);
+};
+
+OlmDevice.prototype.sessionMayHaveProblems = async function(deviceKey, timestamp) {
+    return await this._cryptoStore.getEndToEndSessionProblem(deviceKey, timestamp);
+};
+
+OlmDevice.prototype.filterOutNotifiedErrorDevices = async function(devices) {
+    return await this._cryptoStore.filterOutNotifiedErrorDevices(devices);
+};
+
 
 // Outbound group session
 // ======================
@@ -730,6 +925,8 @@ OlmDevice.prototype.createOutboundGroupSession = function() {
 OlmDevice.prototype.encryptGroupMessage = function(sessionId, payloadString) {
     const self = this;
 
+    logger.log(`encrypting msg with megolm session ${sessionId}`);
+
     checkPayloadLength(payloadString);
 
     return this._getOutboundGroupSession(sessionId, function(session) {
@@ -806,9 +1003,9 @@ OlmDevice.prototype._getInboundGroupSession = function(
     roomId, senderKey, sessionId, txn, func,
 ) {
     this._cryptoStore.getEndToEndInboundGroupSession(
-        senderKey, sessionId, txn, (sessionData) => {
+        senderKey, sessionId, txn, (sessionData, withheld) => {
             if (sessionData === null) {
-                func(null);
+                func(null, null, withheld);
                 return;
             }
 
@@ -822,7 +1019,7 @@ OlmDevice.prototype._getInboundGroupSession = function(
             }
 
             this._unpickleInboundGroupSession(sessionData, (session) => {
-                func(session, sessionData);
+                func(session, sessionData, withheld);
             });
         },
     );
@@ -840,14 +1037,19 @@ OlmDevice.prototype._getInboundGroupSession = function(
  * @param {Object<string, string>} keysClaimed Other keys the sender claims.
  * @param {boolean} exportFormat true if the megolm keys are in export format
  *    (ie, they lack an ed25519 signature)
+ * @param {Object} [extraSessionData={}] any other data to be include with the session
  */
 OlmDevice.prototype.addInboundGroupSession = async function(
     roomId, senderKey, forwardingCurve25519KeyChain,
     sessionId, sessionKey, keysClaimed,
-    exportFormat,
+    exportFormat, extraSessionData = {},
 ) {
     await this._cryptoStore.doTxn(
-        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readwrite', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+            IndexedDBCryptoStore.STORE_SHARED_HISTORY_INBOUND_GROUP_SESSIONS,
+        ], (txn) => {
             /* if we already have this session, consider updating it */
             this._getInboundGroupSession(
                 roomId, senderKey, sessionId, txn,
@@ -873,33 +1075,106 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                                     + senderKey + "/" + sessionId,
                             );
                             if (existingSession.first_known_index()
-                                <= session.first_known_index()) {
+                                <= session.first_known_index()
+                                && !(existingSession.first_known_index() == session.first_known_index()
+                                    && !extraSessionData.untrusted
+                                    && existingSessionData.untrusted)) {
                                 // existing session has lower index (i.e. can
-                                // decrypt more), so keep it
-                                logger.log("Keeping existing session");
+                                // decrypt more), or they have the same index and
+                                // the new sessions trust does not win over the old
+                                // sessions trust, so keep it
+                                logger.log(
+                                    `Keeping existing megolm session ${sessionId}`,
+                                );
                                 return;
                             }
                         }
 
-                        const sessionData = {
+                        logger.info(
+                            "Storing megolm session " + senderKey + "/" + sessionId +
+                            " with first index " + session.first_known_index(),
+                        );
+
+                        const sessionData = Object.assign({}, extraSessionData, {
                             room_id: roomId,
                             session: session.pickle(this._pickleKey),
                             keysClaimed: keysClaimed,
                             forwardingCurve25519KeyChain: forwardingCurve25519KeyChain,
-                        };
+                        });
 
                         this._cryptoStore.storeEndToEndInboundGroupSession(
                             senderKey, sessionId, sessionData, txn,
                         );
+
+                        if (!existingSession && extraSessionData.sharedHistory) {
+                            this._cryptoStore.addSharedHistoryInboundGroupSession(
+                                roomId, senderKey, sessionId, txn,
+                            );
+                        }
                     } finally {
                         session.free();
                     }
                 },
             );
         },
+        logger.withPrefix("[addInboundGroupSession]"),
     );
 };
 
+/**
+ * Record in the data store why an inbound group session was withheld.
+ *
+ * @param {string} roomId     room that the session belongs to
+ * @param {string} senderKey  base64-encoded curve25519 key of the sender
+ * @param {string} sessionId  session identifier
+ * @param {string} code       reason code
+ * @param {string} reason     human-readable version of `code`
+ */
+OlmDevice.prototype.addInboundGroupSessionWithheld = async function(
+    roomId, senderKey, sessionId, code, reason,
+) {
+    await this._cryptoStore.doTxn(
+        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD],
+        (txn) => {
+            this._cryptoStore.storeEndToEndInboundGroupSessionWithheld(
+                senderKey, sessionId,
+                {
+                    room_id: roomId,
+                    code: code,
+                    reason: reason,
+                },
+                txn,
+            );
+        },
+    );
+};
+
+export const WITHHELD_MESSAGES = {
+    "m.unverified": "The sender has disabled encrypting to unverified devices.",
+    "m.blacklisted": "The sender has blocked you.",
+    "m.unauthorised": "You are not authorised to read the message.",
+    "m.no_olm": "Unable to establish a secure channel.",
+};
+
+/**
+ * Calculate the message to use for the exception when a session key is withheld.
+ *
+ * @param {object} withheld  An object that describes why the key was withheld.
+ *
+ * @return {string} the message
+ *
+ * @private
+ */
+function _calculateWithheldMessage(withheld) {
+    if (withheld.code && withheld.code in WITHHELD_MESSAGES) {
+        return WITHHELD_MESSAGES[withheld.code];
+    } else if (withheld.reason) {
+        return withheld.reason;
+    } else {
+        return "decryption key withheld";
+    }
+}
+
 /**
  * Decrypt a received message with an inbound group session
  *
@@ -920,16 +1195,49 @@ OlmDevice.prototype.decryptGroupMessage = async function(
     roomId, senderKey, sessionId, body, eventId, timestamp,
 ) {
     let result;
+    // when the localstorage crypto store is used as an indexeddb backend,
+    // exceptions thrown from within the inner function are not passed through
+    // to the top level, so we store exceptions in a variable and raise them at
+    // the end
+    let error;
 
     await this._cryptoStore.doTxn(
-        'readwrite', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readwrite', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._getInboundGroupSession(
-                roomId, senderKey, sessionId, txn, (session, sessionData) => {
+                roomId, senderKey, sessionId, txn, (session, sessionData, withheld) => {
                     if (session === null) {
+                        if (withheld) {
+                            error = new algorithms.DecryptionError(
+                                "MEGOLM_UNKNOWN_INBOUND_SESSION_ID",
+                                _calculateWithheldMessage(withheld),
+                                {
+                                    session: senderKey + '|' + sessionId,
+                                },
+                            );
+                        }
                         result = null;
                         return;
                     }
-                    const res = session.decrypt(body);
+                    let res;
+                    try {
+                        res = session.decrypt(body);
+                    } catch (e) {
+                        if (e && e.message === 'OLM.UNKNOWN_MESSAGE_INDEX' && withheld) {
+                            error = new algorithms.DecryptionError(
+                                "MEGOLM_UNKNOWN_INBOUND_SESSION_ID",
+                                _calculateWithheldMessage(withheld),
+                                {
+                                    session: senderKey + '|' + sessionId,
+                                },
+                            );
+                        } else {
+                            error = e;
+                        }
+                        return;
+                    }
 
                     let plaintext = res.plaintext;
                     if (plaintext === undefined) {
@@ -951,10 +1259,11 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                                 msgInfo.id !== eventId ||
                                 msgInfo.timestamp !== timestamp
                             ) {
-                                throw new Error(
+                                error = new Error(
                                     "Duplicate message index, possible replay attack: " +
                                     messageIndexKey,
                                 );
+                                return;
                             }
                         }
                         this._inboundGroupSessionMessageIndexes[messageIndexKey] = {
@@ -974,12 +1283,17 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                         forwardingCurve25519KeyChain: (
                             sessionData.forwardingCurve25519KeyChain || []
                         ),
+                        untrusted: sessionData.untrusted,
                     };
                 },
             );
         },
+        logger.withPrefix("[decryptGroupMessage]"),
     );
 
+    if (error) {
+        throw error;
+    }
     return result;
 };
 
@@ -988,14 +1302,17 @@ OlmDevice.prototype.decryptGroupMessage = async function(
  *
  * @param {string} roomId    room in which the message was received
  * @param {string} senderKey base64-encoded curve25519 key of the sender
- * @param {sring} sessionId session identifier
+ * @param {string} sessionId session identifier
  *
  * @returns {Promise<boolean>} true if we have the keys to this session
  */
 OlmDevice.prototype.hasInboundSessionKeys = async function(roomId, senderKey, sessionId) {
     let result;
     await this._cryptoStore.doTxn(
-        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readonly', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._cryptoStore.getEndToEndInboundGroupSession(
                 senderKey, sessionId, txn, (sessionData) => {
                     if (sessionData === null) {
@@ -1017,6 +1334,7 @@ OlmDevice.prototype.hasInboundSessionKeys = async function(roomId, senderKey, se
                 },
             );
         },
+        logger.withPrefix("[hasInboundSessionKeys]"),
     );
 
     return result;
@@ -1046,7 +1364,10 @@ OlmDevice.prototype.getInboundGroupSessionKey = async function(
 ) {
     let result;
     await this._cryptoStore.doTxn(
-        'readonly', [IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS], (txn) => {
+        'readonly', [
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS,
+            IndexedDBCryptoStore.STORE_INBOUND_GROUP_SESSIONS_WITHHELD,
+        ], (txn) => {
             this._getInboundGroupSession(
                 roomId, senderKey, sessionId, txn, (session, sessionData) => {
                     if (session === null) {
@@ -1069,10 +1390,12 @@ OlmDevice.prototype.getInboundGroupSessionKey = async function(
                         "forwarding_curve25519_key_chain":
                             sessionData.forwardingCurve25519KeyChain || [],
                         "sender_claimed_ed25519_key": senderEd25519Key,
+                        "shared_history": sessionData.sharedHistory || false,
                     };
                 },
             );
         },
+        logger.withPrefix("[getInboundGroupSessionKey]"),
     );
 
     return result;
@@ -1100,10 +1423,24 @@ OlmDevice.prototype.exportInboundGroupSession = function(
             "session_key": session.export_session(messageIndex),
             "forwarding_curve25519_key_chain": session.forwardingCurve25519KeyChain || [],
             "first_known_index": session.first_known_index(),
+            "org.matrix.msc3061.shared_history": sessionData.sharedHistory || false,
         };
     });
 };
 
+OlmDevice.prototype.getSharedHistoryInboundGroupSessions = async function(roomId) {
+    let result;
+    await this._cryptoStore.doTxn(
+        'readonly', [
+            IndexedDBCryptoStore.STORE_SHARED_HISTORY_INBOUND_GROUP_SESSIONS,
+        ], (txn) => {
+            result = this._cryptoStore.getSharedHistoryInboundGroupSessions(roomId, txn);
+        },
+        logger.withPrefix("[getSharedHistoryInboundGroupSessionsForRoom]"),
+    );
+    return result;
+};
+
 // Utilities
 // =========
 
@@ -1125,6 +1462,3 @@ OlmDevice.prototype.verifySignature = function(
         util.ed25519_verify(key, message, signature);
     });
 };
-
-/** */
-module.exports = OlmDevice;

src/crypto/OutgoingRoomKeyRequestManager.js

@@ -14,10 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import Promise from 'bluebird';
-
-import logger from '../logger';
-import utils from '../utils';
+import {logger} from '../logger';
+import * as utils from '../utils';
 
 /**
  * Internal module. Management of outgoing room key requests.
@@ -60,7 +58,7 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
  *
  * @enum {number}
  */
-const ROOM_KEY_REQUEST_STATES = {
+export const ROOM_KEY_REQUEST_STATES = {
     /** request not yet sent */
     UNSENT: 0,
 
@@ -77,7 +75,7 @@ const ROOM_KEY_REQUEST_STATES = {
     CANCELLATION_PENDING_AND_WILL_RESEND: 3,
 };
 
-export default class OutgoingRoomKeyRequestManager {
+export class OutgoingRoomKeyRequestManager {
     constructor(baseApis, deviceId, cryptoStore) {
         this._baseApis = baseApis;
         this._deviceId = deviceId;
@@ -99,10 +97,6 @@ export default class OutgoingRoomKeyRequestManager {
      */
     start() {
         this._clientRunning = true;
-
-        // set the timer going, to handle any requests which didn't get sent
-        // on the previous run of the client.
-        this._startTimer();
     }
 
     /**
@@ -115,7 +109,14 @@ export default class OutgoingRoomKeyRequestManager {
     }
 
     /**
-     * Send off a room key request, if we haven't already done so.
+     * Send any requests that have been queued
+     */
+    sendQueuedRequests() {
+        this._startTimer();
+    }
+
+    /**
+     * Queue up a room key request, if we haven't already queued or sent one.
      *
      * The `requestBody` is compared (with a deep-equality check) against
      * previous queued or sent requests and if it matches, no change is made.
@@ -131,7 +132,7 @@ export default class OutgoingRoomKeyRequestManager {
      *    pending list (or we have established that a similar request already
      *    exists)
      */
-    async sendRoomKeyRequest(requestBody, recipients, resend=false) {
+    async queueRoomKeyRequest(requestBody, recipients, resend=false) {
         const req = await this._cryptoStore.getOutgoingRoomKeyRequest(
             requestBody,
         );
@@ -186,7 +187,7 @@ export default class OutgoingRoomKeyRequestManager {
                         // in state ROOM_KEY_REQUEST_STATES.SENT, so we must have
                         // raced with another tab to mark the request cancelled.
                         // Try again, to make sure the request is resent.
-                        return await this.sendRoomKeyRequest(
+                        return await this.queueRoomKeyRequest(
                             requestBody, recipients, resend,
                         );
                     }
@@ -222,9 +223,6 @@ export default class OutgoingRoomKeyRequestManager {
                 throw new Error('unhandled state: ' + req.state);
             }
         }
-        // some of the branches require the timer to be started.  Just start it
-        // all the time, because it doesn't hurt to start it.
-        this._startTimer();
     }
 
     /**
@@ -329,6 +327,21 @@ export default class OutgoingRoomKeyRequestManager {
         );
     }
 
+    /**
+     * Find anything in `sent` state, and kick it around the loop again.
+     * This is intended for situations where something substantial has changed, and we
+     * don't really expect the other end to even care about the cancellation.
+     * For example, after initialization or self-verification.
+     * @return {Promise} An array of `queueRoomKeyRequest` outputs.
+     */
+    async cancelAndResendAllOutgoingRequests() {
+        const outgoings = await this._cryptoStore.getAllOutgoingRoomKeyRequestsByState(
+            ROOM_KEY_REQUEST_STATES.SENT,
+        );
+        return Promise.all(outgoings.map(({ requestBody, recipients }) =>
+            this.queueRoomKeyRequest(requestBody, recipients, true)));
+    }
+
     // start the background timer to send queued requests, if the timer isn't
     // already running
     _startTimer() {
@@ -368,15 +381,12 @@ export default class OutgoingRoomKeyRequestManager {
             return Promise.resolve();
         }
 
-        logger.log("Looking for queued outgoing room key requests");
-
         return this._cryptoStore.getOutgoingRoomKeyRequestByState([
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
             ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
             ROOM_KEY_REQUEST_STATES.UNSENT,
         ]).then((req) => {
             if (!req) {
-                logger.log("No more outgoing room key requests");
                 this._sendOutgoingRoomKeyRequestsTimer = null;
                 return;
             }
@@ -400,7 +410,6 @@ export default class OutgoingRoomKeyRequestManager {
             }).catch((e) => {
                 logger.error("Error sending room key request; will retry later.", e);
                 this._sendOutgoingRoomKeyRequestsTimer = null;
-                this._startTimer();
             });
         });
     }

src/crypto/RoomList.js

@@ -20,12 +20,12 @@ limitations under the License.
  * Manages the list of encrypted rooms
  */
 
-import IndexedDBCryptoStore from './store/indexeddb-crypto-store';
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
 
 /**
  * @alias module:crypto/RoomList
  */
-export default class RoomList {
+export class RoomList {
     constructor(cryptoStore) {
         this._cryptoStore = cryptoStore;
 

src/crypto/SecretStorage.js

@@ -0,0 +1,586 @@
+/*
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {EventEmitter} from 'events';
+import {logger} from '../logger';
+import * as olmlib from './olmlib';
+import {randomString} from '../randomstring';
+import {encryptAES, decryptAES} from './aes';
+import {encodeBase64} from "./olmlib";
+
+export const SECRET_STORAGE_ALGORITHM_V1_AES
+    = "m.secret_storage.v1.aes-hmac-sha2";
+
+const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";
+
+/**
+ * Implements Secure Secret Storage and Sharing (MSC1946)
+ * @module crypto/SecretStorage
+ */
+export class SecretStorage extends EventEmitter {
+    constructor(baseApis, cryptoCallbacks) {
+        super();
+        this._baseApis = baseApis;
+        this._cryptoCallbacks = cryptoCallbacks;
+        this._requests = {};
+        this._incomingRequests = {};
+    }
+
+    async getDefaultKeyId() {
+        const defaultKey = await this._baseApis.getAccountDataFromServer(
+            'm.secret_storage.default_key',
+        );
+        if (!defaultKey) return null;
+        return defaultKey.key;
+    }
+
+    setDefaultKeyId(keyId) {
+        return new Promise(async (resolve, reject) => {
+            const listener = (ev) => {
+                if (
+                    ev.getType() === 'm.secret_storage.default_key' &&
+                    ev.getContent().key === keyId
+                ) {
+                    this._baseApis.removeListener('accountData', listener);
+                    resolve();
+                }
+            };
+            this._baseApis.on('accountData', listener);
+
+            try {
+                await this._baseApis.setAccountData(
+                    'm.secret_storage.default_key',
+                    { key: keyId },
+                );
+            } catch (e) {
+                this._baseApis.removeListener('accountData', listener);
+                reject(e);
+            }
+        });
+    }
+
+    /**
+     * Add a key for encrypting secrets.
+     *
+     * @param {string} algorithm the algorithm used by the key.
+     * @param {object} opts the options for the algorithm.  The properties used
+     *     depend on the algorithm given.
+     * @param {string} [keyId] the ID of the key.  If not given, a random
+     *     ID will be generated.
+     *
+     * @return {object} An object with:
+     *     keyId: {string} the ID of the key
+     *     keyInfo: {object} details about the key (iv, mac, passphrase)
+     */
+    async addKey(algorithm, opts, keyId) {
+        const keyInfo = {algorithm};
+
+        if (!opts) opts = {};
+
+        if (opts.name) {
+            keyInfo.name = opts.name;
+        }
+
+        if (algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (opts.passphrase) {
+                keyInfo.passphrase = opts.passphrase;
+            }
+            if (opts.key) {
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(opts.key);
+                keyInfo.iv = iv;
+                keyInfo.mac = mac;
+            }
+        } else {
+            throw new Error(`Unknown key algorithm ${opts.algorithm}`);
+        }
+
+        if (!keyId) {
+            do {
+                keyId = randomString(32);
+            } while (
+                await this._baseApis.getAccountDataFromServer(
+                    `m.secret_storage.key.${keyId}`,
+                )
+            );
+        }
+
+        await this._baseApis.setAccountData(
+            `m.secret_storage.key.${keyId}`, keyInfo,
+        );
+
+        return {
+            keyId,
+            keyInfo,
+        };
+    }
+
+    /**
+     * Get the key information for a given ID.
+     *
+     * @param {string} [keyId = default key's ID] The ID of the key to check
+     *     for. Defaults to the default key ID if not provided.
+     * @returns {Array?} If the key was found, the return value is an array of
+     *     the form [keyId, keyInfo].  Otherwise, null is returned.
+     */
+    async getKey(keyId) {
+        if (!keyId) {
+            keyId = await this.getDefaultKeyId();
+        }
+        if (!keyId) {
+            return null;
+        }
+
+        const keyInfo = await this._baseApis.getAccountDataFromServer(
+            "m.secret_storage.key." + keyId,
+        );
+        return keyInfo ? [keyId, keyInfo] : null;
+    }
+
+    /**
+     * Check whether we have a key with a given ID.
+     *
+     * @param {string} [keyId = default key's ID] The ID of the key to check
+     *     for. Defaults to the default key ID if not provided.
+     * @return {boolean} Whether we have the key.
+     */
+    async hasKey(keyId) {
+        return !!(await this.getKey(keyId));
+    }
+
+    /**
+     * Check whether a key matches what we expect based on the key info
+     *
+     * @param {Uint8Array} key the key to check
+     * @param {object} info the key info
+     *
+     * @return {boolean} whether or not the key matches
+     */
+    async checkKey(key, info) {
+        if (info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (info.mac) {
+                const {mac} = await SecretStorage._calculateKeyCheck(key, info.iv);
+                return info.mac.replace(/=+$/g, '') === mac.replace(/=+$/g, '');
+            } else {
+                // if we have no information, we have to assume the key is right
+                return true;
+            }
+        } else {
+            throw new Error("Unknown algorithm");
+        }
+    }
+
+    static async _calculateKeyCheck(key, iv) {
+        return await encryptAES(ZERO_STR, key, "", iv);
+    }
+
+    /**
+     * Store an encrypted secret on the server
+     *
+     * @param {string} name The name of the secret
+     * @param {string} secret The secret contents.
+     * @param {Array} keys The IDs of the keys to use to encrypt the secret
+     *     or null/undefined to use the default key.
+     */
+    async store(name, secret, keys) {
+        const encrypted = {};
+
+        if (!keys) {
+            const defaultKeyId = await this.getDefaultKeyId();
+            if (!defaultKeyId) {
+                throw new Error("No keys specified and no default key present");
+            }
+            keys = [defaultKeyId];
+        }
+
+        if (keys.length === 0) {
+            throw new Error("Zero keys given to encrypt with!");
+        }
+
+        for (const keyId of keys) {
+            // get key information from key storage
+            const keyInfo = await this._baseApis.getAccountDataFromServer(
+                "m.secret_storage.key." + keyId,
+            );
+            if (!keyInfo) {
+                throw new Error("Unknown key: " + keyId);
+            }
+
+            // encrypt secret, based on the algorithm
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                const keys = {[keyId]: keyInfo};
+                const [, encryption] = await this._getSecretStorageKey(keys, name);
+                encrypted[keyId] = await encryption.encrypt(secret);
+            } else {
+                logger.warn("unknown algorithm for secret storage key " + keyId
+                            + ": " + keyInfo.algorithm);
+                // do nothing if we don't understand the encryption algorithm
+            }
+        }
+
+        // save encrypted secret
+        await this._baseApis.setAccountData(name, {encrypted});
+    }
+
+    /**
+     * Temporary method to fix up existing accounts where secrets
+     * are incorrectly stored without the 'encrypted' level
+     *
+     * @param {string} name The name of the secret
+     * @param {object} secretInfo The account data object
+     * @returns {object} The fixed object or null if no fix was performed
+     */
+    async _fixupStoredSecret(name, secretInfo) {
+        // We assume the secret was only stored passthrough for 1
+        // key - this was all the broken code supported.
+        const keys = Object.keys(secretInfo);
+        if (
+            keys.length === 1 && keys[0] !== 'encrypted' &&
+            secretInfo[keys[0]].passthrough
+        ) {
+            const hasKey = await this.hasKey(keys[0]);
+            if (hasKey) {
+                logger.log("Fixing up passthrough secret: " + name);
+                await this.storePassthrough(name, keys[0]);
+                const newData = await this._baseApis.getAccountDataFromServer(name);
+                return newData;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Get a secret from storage.
+     *
+     * @param {string} name the name of the secret
+     *
+     * @return {string} the contents of the secret
+     */
+    async get(name) {
+        let secretInfo = await this._baseApis.getAccountDataFromServer(name);
+        if (!secretInfo) {
+            return;
+        }
+        if (!secretInfo.encrypted) {
+            // try to fix it up
+            secretInfo = await this._fixupStoredSecret(name, secretInfo);
+            if (!secretInfo || !secretInfo.encrypted) {
+                throw new Error("Content is not encrypted!");
+            }
+        }
+
+        // get possible keys to decrypt
+        const keys = {};
+        for (const keyId of Object.keys(secretInfo.encrypted)) {
+            // get key information from key storage
+            const keyInfo = await this._baseApis.getAccountDataFromServer(
+                "m.secret_storage.key." + keyId,
+            );
+            const encInfo = secretInfo.encrypted[keyId];
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    keys[keyId] = keyInfo;
+                }
+            }
+        }
+
+        if (Object.keys(keys).length === 0) {
+            throw new Error(`Could not decrypt ${name} because none of ` +
+                `the keys it is encrypted with are for a supported algorithm`);
+        }
+
+        let keyId;
+        let decryption;
+        try {
+            // fetch private key from app
+            [keyId, decryption] = await this._getSecretStorageKey(keys, name);
+
+            const encInfo = secretInfo.encrypted[keyId];
+
+            // We don't actually need the decryption object if it's a passthrough
+            // since we just want to return the key itself. It must be base64
+            // encoded, since this is how a key would normally be stored.
+            if (encInfo.passthrough) return encodeBase64(decryption.get_private_key());
+
+            return await decryption.decrypt(encInfo);
+        } finally {
+            if (decryption && decryption.free) decryption.free();
+        }
+    }
+
+    /**
+     * Check if a secret is stored on the server.
+     *
+     * @param {string} name the name of the secret
+     * @param {boolean} checkKey check if the secret is encrypted by a trusted key
+     *
+     * @return {object?} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
+     */
+    async isStored(name, checkKey) {
+        // check if secret exists
+        let secretInfo = await this._baseApis.getAccountDataFromServer(name);
+        if (!secretInfo) return null;
+        if (!secretInfo.encrypted) {
+            // try to fix it up
+            secretInfo = await this._fixupStoredSecret(name, secretInfo);
+            if (!secretInfo || !secretInfo.encrypted) {
+                return null;
+            }
+        }
+
+        if (checkKey === undefined) checkKey = true;
+
+        const ret = {};
+
+        // filter secret encryption keys with supported algorithm
+        for (const keyId of Object.keys(secretInfo.encrypted)) {
+            // get key information from key storage
+            const keyInfo = await this._baseApis.getAccountDataFromServer(
+                "m.secret_storage.key." + keyId,
+            );
+            if (!keyInfo) continue;
+            const encInfo = secretInfo.encrypted[keyId];
+
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    ret[keyId] = keyInfo;
+                }
+            }
+        }
+        return Object.keys(ret).length ? ret : null;
+    }
+
+    /**
+     * Request a secret from another device
+     *
+     * @param {string} name the name of the secret to request
+     * @param {string[]} devices the devices to request the secret from
+     *
+     * @return {string} the contents of the secret
+     */
+    request(name, devices) {
+        const requestId = this._baseApis.makeTxnId();
+
+        const requestControl = this._requests[requestId] = {
+            name,
+            devices,
+        };
+        const promise = new Promise((resolve, reject) => {
+            requestControl.resolve = resolve;
+            requestControl.reject = reject;
+        });
+        const cancel = (reason) => {
+            // send cancellation event
+            const cancelData = {
+                action: "request_cancellation",
+                requesting_device_id: this._baseApis.deviceId,
+                request_id: requestId,
+            };
+            const toDevice = {};
+            for (const device of devices) {
+                toDevice[device] = cancelData;
+            }
+            this._baseApis.sendToDevice("m.secret.request", {
+                [this._baseApis.getUserId()]: toDevice,
+            });
+
+            // and reject the promise so that anyone waiting on it will be
+            // notified
+            requestControl.reject(new Error(reason || "Cancelled"));
+        };
+
+        // send request to devices
+        const requestData = {
+            name,
+            action: "request",
+            requesting_device_id: this._baseApis.deviceId,
+            request_id: requestId,
+        };
+        const toDevice = {};
+        for (const device of devices) {
+            toDevice[device] = requestData;
+        }
+        logger.info(`Request secret ${name} from ${devices}, id ${requestId}`);
+        this._baseApis.sendToDevice("m.secret.request", {
+            [this._baseApis.getUserId()]: toDevice,
+        });
+
+        return {
+            request_id: requestId,
+            promise,
+            cancel,
+        };
+    }
+
+    async _onRequestReceived(event) {
+        const sender = event.getSender();
+        const content = event.getContent();
+        if (sender !== this._baseApis.getUserId()
+            || !(content.name && content.action
+                 && content.requesting_device_id && content.request_id)) {
+            // ignore requests from anyone else, for now
+            return;
+        }
+        const deviceId = content.requesting_device_id;
+        // check if it's a cancel
+        if (content.action === "request_cancellation") {
+            if (this._incomingRequests[deviceId]
+                && this._incomingRequests[deviceId][content.request_id]) {
+                logger.info("received request cancellation for secret (" + sender
+                            + ", " + deviceId + ", " + content.request_id + ")");
+                this.baseApis.emit("crypto.secrets.requestCancelled", {
+                    user_id: sender,
+                    device_id: deviceId,
+                    request_id: content.request_id,
+                });
+            }
+        } else if (content.action === "request") {
+            if (deviceId === this._baseApis.deviceId) {
+                // no point in trying to send ourself the secret
+                return;
+            }
+
+            // check if we have the secret
+            logger.info("received request for secret (" + sender
+                        + ", " + deviceId + ", " + content.request_id + ")");
+            if (!this._cryptoCallbacks.onSecretRequested) {
+                return;
+            }
+            const secret = await this._cryptoCallbacks.onSecretRequested(
+                sender,
+                deviceId,
+                content.request_id,
+                content.name,
+                this._baseApis.checkDeviceTrust(sender, deviceId),
+            );
+            if (secret) {
+                logger.info(`Preparing ${content.name} secret for ${deviceId}`);
+                const payload = {
+                    type: "m.secret.send",
+                    content: {
+                        request_id: content.request_id,
+                        secret: secret,
+                    },
+                };
+                const encryptedContent = {
+                    algorithm: olmlib.OLM_ALGORITHM,
+                    sender_key: this._baseApis._crypto._olmDevice.deviceCurve25519Key,
+                    ciphertext: {},
+                };
+                await olmlib.ensureOlmSessionsForDevices(
+                    this._baseApis._crypto._olmDevice,
+                    this._baseApis,
+                    {
+                        [sender]: [
+                            this._baseApis.getStoredDevice(sender, deviceId),
+                        ],
+                    },
+                );
+                await olmlib.encryptMessageForDevice(
+                    encryptedContent.ciphertext,
+                    this._baseApis.getUserId(),
+                    this._baseApis.deviceId,
+                    this._baseApis._crypto._olmDevice,
+                    sender,
+                    this._baseApis.getStoredDevice(sender, deviceId),
+                    payload,
+                );
+                const contentMap = {
+                    [sender]: {
+                        [deviceId]: encryptedContent,
+                    },
+                };
+
+                logger.info(`Sending ${content.name} secret for ${deviceId}`);
+                this._baseApis.sendToDevice("m.room.encrypted", contentMap);
+            } else {
+                logger.info(`Request denied for ${content.name} secret for ${deviceId}`);
+            }
+        }
+    }
+
+    _onSecretReceived(event) {
+        if (event.getSender() !== this._baseApis.getUserId()) {
+            // we shouldn't be receiving secrets from anyone else, so ignore
+            // because someone could be trying to send us bogus data
+            return;
+        }
+        const content = event.getContent();
+        logger.log("got secret share for request", content.request_id);
+        const requestControl = this._requests[content.request_id];
+        if (requestControl) {
+            // make sure that the device that sent it is one of the devices that
+            // we requested from
+            const deviceInfo = this._baseApis._crypto._deviceList.getDeviceByIdentityKey(
+                olmlib.OLM_ALGORITHM,
+                event.getSenderKey(),
+            );
+            if (!deviceInfo) {
+                logger.log(
+                    "secret share from unknown device with key", event.getSenderKey(),
+                );
+                return;
+            }
+            if (!requestControl.devices.includes(deviceInfo.deviceId)) {
+                logger.log("unsolicited secret share from device", deviceInfo.deviceId);
+                return;
+            }
+
+            logger.log(
+                `Successfully received secret ${requestControl.name} ` +
+                `from ${deviceInfo.deviceId}`,
+            );
+            requestControl.resolve(content.secret);
+        }
+    }
+
+    async _getSecretStorageKey(keys, name) {
+        if (!this._cryptoCallbacks.getSecretStorageKey) {
+            throw new Error("No getSecretStorageKey callback supplied");
+        }
+
+        const returned = await this._cryptoCallbacks.getSecretStorageKey({ keys }, name);
+
+        if (!returned) {
+            throw new Error("getSecretStorageKey callback returned falsey");
+        }
+        if (returned.length < 2) {
+            throw new Error("getSecretStorageKey callback returned invalid data");
+        }
+
+        const [keyId, privateKey] = returned;
+        if (!keys[keyId]) {
+            throw new Error("App returned unknown key from getSecretStorageKey!");
+        }
+
+        if (keys[keyId].algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            const decryption = {
+                encrypt: async function(secret) {
+                    return await encryptAES(secret, privateKey, name);
+                },
+                decrypt: async function(encInfo) {
+                    return await decryptAES(encInfo, privateKey, name);
+                },
+            };
+            return [keyId, decryption];
+        } else {
+            throw new Error("Unknown key type: " + keys[keyId].algorithm);
+        }
+    }
+}

src/crypto/aes.js

@@ -0,0 +1,251 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {getCrypto} from '../utils';
+import {decodeBase64, encodeBase64} from './olmlib';
+
+const subtleCrypto = (typeof window !== "undefined" && window.crypto) ?
+    (window.crypto.subtle || window.crypto.webkitSubtle) : null;
+
+// salt for HKDF, with 8 bytes of zeros
+const zerosalt = new Uint8Array(8);
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptNode(data, key, name, ivStr) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = crypto.randomBytes(16);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const cipher = crypto.createCipheriv("aes-256-ctr", aesKey, iv);
+    const ciphertext = cipher.update(data, "utf-8", "base64")
+          + cipher.final("base64");
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(ciphertext, "base64").digest("base64");
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: ciphertext,
+        mac: hmac,
+    };
+}
+
+/**
+ * decrypt a string in Node.js
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptNode(data, key, name) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(data.ciphertext, "base64").digest("base64").replace(/=+$/g, '');
+
+    if (hmac !== data.mac.replace(/=+$/g, '')) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const decipher = crypto.createDecipheriv(
+        "aes-256-ctr", aesKey, decodeBase64(data.iv),
+    );
+    return decipher.update(data.ciphertext, "base64", "utf-8")
+          + decipher.final("utf-8");
+}
+
+function deriveKeysNode(key, name) {
+    const crypto = getCrypto();
+    const prk = crypto.createHmac("sha256", zerosalt)
+        .update(key).digest();
+
+    const b = Buffer.alloc(1, 1);
+    const aesKey = crypto.createHmac("sha256", prk)
+        .update(name, "utf-8").update(b).digest();
+    b[0] = 2;
+    const hmacKey = crypto.createHmac("sha256", prk)
+        .update(aesKey).update(name, "utf-8").update(b).digest();
+
+    return [aesKey, hmacKey];
+}
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
+ */
+async function encryptBrowser(data, key, name, ivStr) {
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = new Uint8Array(16);
+        window.crypto.getRandomValues(iv);
+    }
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+    const encodedData = new TextEncoder().encode(data);
+
+    const ciphertext = await subtleCrypto.encrypt(
+        {
+            name: "AES-CTR",
+            counter: iv,
+            length: 64,
+        },
+        aesKey,
+        encodedData,
+    );
+
+    const hmac = await subtleCrypto.sign(
+        {name: 'HMAC'},
+        hmacKey,
+        ciphertext,
+    );
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: encodeBase64(ciphertext),
+        mac: encodeBase64(hmac),
+    };
+}
+
+/**
+ * decrypt a string in the browser
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptBrowser(data, key, name) {
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+
+    const ciphertext = decodeBase64(data.ciphertext);
+
+    if (!await subtleCrypto.verify(
+        {name: "HMAC"},
+        hmacKey,
+        decodeBase64(data.mac),
+        ciphertext,
+    )) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const plaintext = await subtleCrypto.decrypt(
+        {
+            name: "AES-CTR",
+            counter: decodeBase64(data.iv),
+            length: 64,
+        },
+        aesKey,
+        ciphertext,
+    );
+
+    return new TextDecoder().decode(new Uint8Array(plaintext));
+}
+
+async function deriveKeysBrowser(key, name) {
+    const hkdfkey = await subtleCrypto.importKey(
+        'raw',
+        key,
+        {name: "HKDF"},
+        false,
+        ["deriveBits"],
+    );
+    const keybits = await subtleCrypto.deriveBits(
+        {
+            name: "HKDF",
+            salt: zerosalt,
+            info: (new TextEncoder().encode(name)),
+            hash: "SHA-256",
+        },
+        hkdfkey,
+        512,
+    );
+
+    const aesKey = keybits.slice(0, 32);
+    const hmacKey = keybits.slice(32);
+
+    const aesProm = subtleCrypto.importKey(
+        'raw',
+        aesKey,
+        {name: 'AES-CTR'},
+        false,
+        ['encrypt', 'decrypt'],
+    );
+
+    const hmacProm = subtleCrypto.importKey(
+        'raw',
+        hmacKey,
+        {
+            name: 'HMAC',
+            hash: {name: 'SHA-256'},
+        },
+        false,
+        ['sign', 'verify'],
+    );
+
+    return await Promise.all([aesProm, hmacProm]);
+}
+
+export function encryptAES(...args) {
+    return subtleCrypto ? encryptBrowser(...args) : encryptNode(...args);
+}
+
+export function decryptAES(...args) {
+    return subtleCrypto ? decryptBrowser(...args) : decryptNode(...args);
+}
+

src/crypto/algorithms/base.js

@@ -20,8 +20,6 @@ limitations under the License.
  * @module
  */
 
-import Promise from 'bluebird';
-
 /**
  * map of registered encryption algorithm classes. A map from string to {@link
  * module:crypto/algorithms/base.EncryptionAlgorithm|EncryptionAlgorithm} class
@@ -52,7 +50,7 @@ export const DECRYPTION_CLASSES = {};
  * @param {string} params.roomId  The ID of the room we will be sending to
  * @param {object} params.config  The body of the m.room.encryption event
  */
-class EncryptionAlgorithm {
+export class EncryptionAlgorithm {
     constructor(params) {
         this._userId = params.userId;
         this._deviceId = params.deviceId;
@@ -62,6 +60,15 @@ class EncryptionAlgorithm {
         this._roomId = params.roomId;
     }
 
+    /**
+     * Perform any background tasks that can be done before a message is ready to
+     * send, in order to speed up sending of the message.
+     *
+     * @param {module:models/room} room the room the event is in
+     */
+    prepareToEncrypt(room) {
+    }
+
     /**
      * Encrypt a message event
      *
@@ -72,7 +79,7 @@ class EncryptionAlgorithm {
      * @param {string} eventType
      * @param {object} plaintext event content
      *
-     * @return {module:client.Promise} Promise which resolves to the new event body
+     * @return {Promise} Promise which resolves to the new event body
      */
 
     /**
@@ -86,7 +93,6 @@ class EncryptionAlgorithm {
      onRoomMembership(event, member, oldMembership) {
      }
 }
-export {EncryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * base type for decryption implementations
@@ -100,7 +106,7 @@ export {EncryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
  * @param {string=} params.roomId The ID of the room we will be receiving
  *     from. Null for to-device events.
  */
-class DecryptionAlgorithm {
+export class DecryptionAlgorithm {
     constructor(params) {
         this._userId = params.userId;
         this._crypto = params.crypto;
@@ -161,8 +167,17 @@ class DecryptionAlgorithm {
     shareKeysWithDevice(keyRequest) {
         throw new Error("shareKeysWithDevice not supported for this DecryptionAlgorithm");
     }
+
+    /**
+     * Retry decrypting all the events from a sender that haven't been
+     * decrypted yet.
+     *
+     * @param {string} senderKey the sender's key
+     */
+    async retryDecryptionFromSender(senderKey) {
+        // ignore by default
+    }
 }
-export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 /**
  * Exception thrown when decryption fails
@@ -175,7 +190,7 @@ export {DecryptionAlgorithm}; // https://github.com/jsdoc3/jsdoc/issues/1272
  *
  * @extends Error
  */
-class DecryptionError extends Error {
+export class DecryptionError extends Error {
     constructor(code, msg, details) {
         super(msg);
         this.code = code;
@@ -183,7 +198,6 @@ class DecryptionError extends Error {
         this.detailedString = _detailedStringForDecryptionError(this, details);
     }
 }
-export {DecryptionError}; // https://github.com/jsdoc3/jsdoc/issues/1272
 
 function _detailedStringForDecryptionError(err, details) {
     let result = err.name + '[msg: ' + err.message;

src/crypto/algorithms/index.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,28 +14,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /**
  * @module crypto/algorithms
  */
 
-const base = require("./base");
+import "./olm";
+import "./megolm";
 
-require("./olm");
-require("./megolm");
-
-/**
- * @see module:crypto/algorithms/base.ENCRYPTION_CLASSES
- */
-module.exports.ENCRYPTION_CLASSES = base.ENCRYPTION_CLASSES;
-
-/**
- * @see module:crypto/algorithms/base.DECRYPTION_CLASSES
- */
-module.exports.DECRYPTION_CLASSES = base.DECRYPTION_CLASSES;
-
-/**
- * @see module:crypto/algorithms/base.DecryptionError
- */
-module.exports.DecryptionError = base.DecryptionError;
+export * from "./base";

src/crypto/algorithms/olm.js

@@ -13,45 +13,48 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
 
 /**
  * Defines m.olm encryption/decryption
  *
  * @module crypto/algorithms/olm
  */
-import Promise from 'bluebird';
 
-import logger from '../../logger';
-const utils = require("../../utils");
-const olmlib = require("../olmlib");
-const DeviceInfo = require("../deviceinfo");
+import {logger} from '../../logger';
+import * as utils from "../../utils";
+import {polyfillSuper} from "../../utils";
+import * as olmlib from "../olmlib";
+import {DeviceInfo} from "../deviceinfo";
+import {
+    DecryptionAlgorithm,
+    DecryptionError,
+    EncryptionAlgorithm,
+    registerAlgorithm,
+} from "./base";
+
 const DeviceVerification = DeviceInfo.DeviceVerification;
 
-
-const base = require("./base");
-
 /**
  * Olm encryption implementation
  *
  * @constructor
- * @extends {module:crypto/algorithms/base.EncryptionAlgorithm}
+ * @extends {module:crypto/algorithms/EncryptionAlgorithm}
  *
  * @param {object} params parameters, as per
- *     {@link module:crypto/algorithms/base.EncryptionAlgorithm}
+ *     {@link module:crypto/algorithms/EncryptionAlgorithm}
  */
 function OlmEncryption(params) {
-    base.EncryptionAlgorithm.call(this, params);
+    polyfillSuper(this, EncryptionAlgorithm, params);
     this._sessionPrepared = false;
     this._prepPromise = null;
 }
-utils.inherits(OlmEncryption, base.EncryptionAlgorithm);
+utils.inherits(OlmEncryption, EncryptionAlgorithm);
 
 /**
  * @private
 
  * @param {string[]} roomMembers list of currently-joined users in the room
- * @return {module:client.Promise} Promise which resolves when setup is complete
+ * @return {Promise} Promise which resolves when setup is complete
  */
 OlmEncryption.prototype._ensureSession = function(roomMembers) {
     if (this._prepPromise) {
@@ -82,7 +85,7 @@ OlmEncryption.prototype._ensureSession = function(roomMembers) {
  * @param {string} eventType
  * @param {object} content plaintext event content
  *
- * @return {module:client.Promise} Promise which resolves to the new event body
+ * @return {Promise} Promise which resolves to the new event body
  */
 OlmEncryption.prototype.encryptMessage = async function(room, eventType, content) {
     // pick the list of recipients based on the membership list.
@@ -139,21 +142,21 @@ OlmEncryption.prototype.encryptMessage = async function(room, eventType, content
         }
     }
 
-    return await Promise.all(promises).return(encryptedContent);
+    return await Promise.all(promises).then(() => encryptedContent);
 };
 
 /**
  * Olm decryption implementation
  *
  * @constructor
- * @extends {module:crypto/algorithms/base.DecryptionAlgorithm}
+ * @extends {module:crypto/algorithms/DecryptionAlgorithm}
  * @param {object} params parameters, as per
- *     {@link module:crypto/algorithms/base.DecryptionAlgorithm}
+ *     {@link module:crypto/algorithms/DecryptionAlgorithm}
  */
 function OlmDecryption(params) {
-    base.DecryptionAlgorithm.call(this, params);
+    polyfillSuper(this, DecryptionAlgorithm, params);
 }
-utils.inherits(OlmDecryption, base.DecryptionAlgorithm);
+utils.inherits(OlmDecryption, DecryptionAlgorithm);
 
 /**
  * @inheritdoc
@@ -171,14 +174,14 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
     const ciphertext = content.ciphertext;
 
     if (!ciphertext) {
-        throw new base.DecryptionError(
+        throw new DecryptionError(
             "OLM_MISSING_CIPHERTEXT",
             "Missing ciphertext",
         );
     }
 
     if (!(this._olmDevice.deviceCurve25519Key in ciphertext)) {
-        throw new base.DecryptionError(
+        throw new DecryptionError(
             "OLM_NOT_INCLUDED_IN_RECIPIENTS",
             "Not included in recipients",
         );
@@ -189,7 +192,7 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
     try {
         payloadString = await this._decryptMessage(deviceKey, message);
     } catch (e) {
-        throw new base.DecryptionError(
+        throw new DecryptionError(
             "OLM_BAD_ENCRYPTED_MESSAGE",
             "Bad Encrypted Message", {
                 sender: deviceKey,
@@ -203,14 +206,14 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
     // check that we were the intended recipient, to avoid unknown-key attack
     // https://github.com/vector-im/vector-web/issues/2483
     if (payload.recipient != this._userId) {
-        throw new base.DecryptionError(
+        throw new DecryptionError(
             "OLM_BAD_RECIPIENT",
             "Message was intented for " + payload.recipient,
         );
     }
 
     if (payload.recipient_keys.ed25519 != this._olmDevice.deviceEd25519Key) {
-        throw new base.DecryptionError(
+        throw new DecryptionError(
             "OLM_BAD_RECIPIENT_KEY",
             "Message not intended for this device", {
                 intended: payload.recipient_keys.ed25519,
@@ -224,7 +227,7 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
     // (this check is also provided via the sender's embedded ed25519 key,
     // which is checked elsewhere).
     if (payload.sender != event.getSender()) {
-        throw new base.DecryptionError(
+        throw new DecryptionError(
             "OLM_FORWARDED_MESSAGE",
             "Message forwarded from " + payload.sender, {
                 reported_sender: event.getSender(),
@@ -234,7 +237,7 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
 
     // Olm events intended for a room have a room_id.
     if (payload.room_id !== event.getRoomId()) {
-        throw new base.DecryptionError(
+        throw new DecryptionError(
             "OLM_BAD_ROOM",
             "Message intended for room " + payload.room_id, {
                 reported_room: event.room_id,
@@ -261,6 +264,25 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
  */
 OlmDecryption.prototype._decryptMessage = async function(
     theirDeviceIdentityKey, message,
+) {
+    // This is a wrapper that serialises decryptions of prekey messages, because
+    // otherwise we race between deciding we have no active sessions for the message
+    // and creating a new one, which we can only do once because it removes the OTK.
+    if (message.type !== 0) {
+        // not a prekey message: we can safely just try & decrypt it
+        return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+    } else {
+        const myPromise = this._olmDevice._olmPrekeyPromise.then(() => {
+            return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+        });
+        // we want the error, but don't propagate it to the next decryption
+        this._olmDevice._olmPrekeyPromise = myPromise.catch(() => {});
+        return await myPromise;
+    }
+};
+
+OlmDecryption.prototype._reallyDecryptMessage = async function(
+    theirDeviceIdentityKey, message,
 ) {
     const sessionIds = await this._olmDevice.getSessionIdsForDevice(
         theirDeviceIdentityKey,
@@ -337,4 +359,4 @@ OlmDecryption.prototype._decryptMessage = async function(
 };
 
 
-base.registerAlgorithm(olmlib.OLM_ALGORITHM, OlmEncryption, OlmDecryption);
+registerAlgorithm(olmlib.OLM_ALGORITHM, OlmEncryption, OlmDecryption);

src/crypto/dehydration.ts

@@ -0,0 +1,288 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {decodeBase64, encodeBase64} from './olmlib';
+import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+import anotherjson from "another-json";
+import {logger} from '../logger';
+
+// FIXME: these types should eventually go in a different file
+type Signatures = Record<string, Record<string, string>>;
+
+interface DeviceKeys {
+    algorithms: Array<string>;
+    device_id: string; // eslint-disable-line camelcase
+    user_id: string; // eslint-disable-line camelcase
+    keys: Record<string, string>;
+    signatures?: Signatures;
+}
+
+interface OneTimeKey {
+    key: string;
+    fallback?: boolean;
+    signatures?: Signatures;
+}
+
+export const DEHYDRATION_ALGORITHM = "org.matrix.msc2697.v1.olm.libolm_pickle";
+
+const oneweek = 7 * 24 * 60 * 60 * 1000;
+
+export class DehydrationManager {
+    private inProgress = false;
+    private timeoutId: any;
+    private key: Uint8Array;
+    private keyInfo: {[props: string]: any};
+    private deviceDisplayName: string;
+    constructor(private crypto) {
+        this.getDehydrationKeyFromCache();
+    }
+    async getDehydrationKeyFromCache(): Promise<void> {
+        return await this.crypto._cryptoStore.doTxn(
+            'readonly',
+            [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                this.crypto._cryptoStore.getSecretStorePrivateKey(
+                    txn,
+                    async (result) => {
+                        if (result) {
+                            const {key, keyInfo, deviceDisplayName, time} = result;
+                            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+                            const decrypted = await decryptAES(key, pickleKey, DEHYDRATION_ALGORITHM);
+                            this.key = decodeBase64(decrypted);
+                            this.keyInfo = keyInfo;
+                            this.deviceDisplayName = deviceDisplayName;
+                            const now = Date.now();
+                            const delay = Math.max(1, time + oneweek - now);
+                            this.timeoutId = global.setTimeout(
+                                this.dehydrateDevice.bind(this), delay,
+                            );
+                        }
+                    },
+                    "dehydration",
+                );
+            },
+        );
+    }
+
+    /** set the key, and queue periodic dehydration to the server in the background */
+    async setKeyAndQueueDehydration(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<void> {
+        const matches = await this.setKey(key, keyInfo, deviceDisplayName);
+        if (!matches) {
+            // start dehydration in the background
+            this.dehydrateDevice();
+        }
+    }
+
+    async setKey(
+        key: Uint8Array, keyInfo: {[props: string]: any} = {},
+        deviceDisplayName: string = undefined,
+    ): Promise<boolean> {
+        if (!key) {
+            // unsetting the key -- cancel any pending dehydration task
+            if (this.timeoutId) {
+                global.clearTimeout(this.timeoutId);
+                this.timeoutId = undefined;
+            }
+            // clear storage
+            await this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration", null,
+                    );
+                },
+            );
+            this.key = undefined;
+            this.keyInfo = undefined;
+            return;
+        }
+
+        // Check to see if it's the same key as before.  If it's different,
+        // dehydrate a new device.  If it's the same, we can keep the same
+        // device.  (Assume that keyInfo and deviceDisplayName will be the
+        // same if the key is the same.)
+        let matches: boolean = this.key && key.length == this.key.length;
+        for (let i = 0; matches && i < key.length; i++) {
+            if (key[i] != this.key[i]) {
+                matches = false;
+            }
+        }
+        if (!matches) {
+            this.key = key;
+            this.keyInfo = keyInfo;
+            this.deviceDisplayName = deviceDisplayName;
+        }
+        return matches;
+    }
+
+    /** returns the device id of the newly created dehydrated device */
+    async dehydrateDevice(): Promise<string> {
+        if (this.inProgress) {
+            logger.log("Dehydration already in progress -- not starting new dehydration");
+            return;
+        }
+        this.inProgress = true;
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+        try {
+            const pickleKey = Buffer.from(this.crypto._olmDevice._pickleKey);
+
+            // update the crypto store with the timestamp
+            const key = await encryptAES(encodeBase64(this.key), pickleKey, DEHYDRATION_ALGORITHM);
+            await this.crypto._cryptoStore.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    this.crypto._cryptoStore.storeSecretStorePrivateKey(
+                        txn, "dehydration",
+                        {
+                            keyInfo: this.keyInfo,
+                            key,
+                            deviceDisplayName: this.deviceDisplayName,
+                            time: Date.now(),
+                        },
+                    );
+                },
+            );
+            logger.log("Attempting to dehydrate device");
+
+            logger.log("Creating account");
+            // create the account and all the necessary keys
+            const account = new global.Olm.Account();
+            account.create();
+            const e2eKeys = JSON.parse(account.identity_keys());
+
+            const maxKeys = account.max_number_of_one_time_keys();
+            // FIXME: generate in small batches?
+            account.generate_one_time_keys(maxKeys / 2);
+            account.generate_fallback_key();
+            const otks: Record<string, string> = JSON.parse(account.one_time_keys());
+            const fallbacks: Record<string, string> = JSON.parse(account.fallback_key());
+            account.mark_keys_as_published();
+
+            // dehydrate the account and store it on the server
+            const pickledAccount = account.pickle(new Uint8Array(this.key));
+
+            const deviceData: {[props: string]: any} = {
+                algorithm: DEHYDRATION_ALGORITHM,
+                account: pickledAccount,
+            };
+            if (this.keyInfo.passphrase) {
+                deviceData.passphrase = this.keyInfo.passphrase;
+            }
+
+            logger.log("Uploading account to server");
+            const dehydrateResult = await this.crypto._baseApis._http.authedRequest(
+                undefined,
+                "PUT",
+                "/dehydrated_device",
+                undefined,
+                {
+                    device_data: deviceData,
+                    initial_device_display_name: this.deviceDisplayName,
+                },
+                {
+                    prefix: "/_matrix/client/unstable/org.matrix.msc2697.v2",
+                },
+            );
+
+            // send the keys to the server
+            const deviceId = dehydrateResult.device_id;
+            logger.log("Preparing device keys", deviceId);
+            const deviceKeys: DeviceKeys = {
+                algorithms: this.crypto._supportedAlgorithms,
+                device_id: deviceId,
+                user_id: this.crypto._userId,
+                keys: {
+                    [`ed25519:${deviceId}`]: e2eKeys.ed25519,
+                    [`curve25519:${deviceId}`]: e2eKeys.curve25519,
+                },
+            };
+            const deviceSignature = account.sign(anotherjson.stringify(deviceKeys));
+            deviceKeys.signatures = {
+                [this.crypto._userId]: {
+                    [`ed25519:${deviceId}`]: deviceSignature,
+                },
+            };
+            if (this.crypto._crossSigningInfo.getId("self_signing")) {
+                await this.crypto._crossSigningInfo.signObject(deviceKeys, "self_signing");
+            }
+
+            logger.log("Preparing one-time keys");
+            const oneTimeKeys = {};
+            for (const [keyId, key] of Object.entries(otks.curve25519)) {
+                const k: OneTimeKey = {key};
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                oneTimeKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Preparing fallback keys");
+            const fallbackKeys = {};
+            for (const [keyId, key] of Object.entries(fallbacks.curve25519)) {
+                const k: OneTimeKey = {key, fallback: true};
+                const signature = account.sign(anotherjson.stringify(k));
+                k.signatures = {
+                    [this.crypto._userId]: {
+                        [`ed25519:${deviceId}`]: signature,
+                    },
+                };
+                fallbackKeys[`signed_curve25519:${keyId}`] = k;
+            }
+
+            logger.log("Uploading keys to server");
+            await this.crypto._baseApis._http.authedRequest(
+                undefined,
+                "POST",
+                "/keys/upload/" + encodeURI(deviceId),
+                undefined,
+                {
+                    "device_keys": deviceKeys,
+                    "one_time_keys": oneTimeKeys,
+                    "org.matrix.msc2732.fallback_keys": fallbackKeys,
+                },
+            );
+            logger.log("Done dehydrating");
+
+            // dehydrate again in a week
+            this.timeoutId = global.setTimeout(
+                this.dehydrateDevice.bind(this), oneweek,
+            );
+
+            return deviceId;
+        } finally {
+            this.inProgress = false;
+        }
+    }
+
+    private stop() {
+        if (this.timeoutId) {
+            global.clearTimeout(this.timeoutId);
+            this.timeoutId = undefined;
+        }
+    }
+}

src/crypto/deviceinfo.js

@@ -1,5 +1,6 @@
 /*
 Copyright 2016 OpenMarket Ltd
+Copyright 2019 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,8 +14,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-"use strict";
-
 
 /**
  * @module crypto/deviceinfo
@@ -44,7 +43,7 @@ limitations under the License.
   *
   * @param {string} deviceId id of the device
   */
-function DeviceInfo(deviceId) {
+export function DeviceInfo(deviceId) {
     // you can't change the deviceId
     Object.defineProperty(this, 'deviceId', {
         enumerable: true,
@@ -56,6 +55,7 @@ function DeviceInfo(deviceId) {
     this.verified = DeviceVerification.UNVERIFIED;
     this.known = false;
     this.unsigned = {};
+    this.signatures = {};
 }
 
 /**
@@ -88,6 +88,7 @@ DeviceInfo.prototype.toStorage = function() {
         verified: this.verified,
         known: this.known,
         unsigned: this.unsigned,
+        signatures: this.signatures,
     };
 };
 
@@ -165,5 +166,3 @@ DeviceInfo.DeviceVerification = {
 
 const DeviceVerification = DeviceInfo.DeviceVerification;
 
-/** */
-module.exports = DeviceInfo;