v5.2.0

Fix isVerified returning false

.buildkite/pipeline.yaml

@@ -1,58 +0,0 @@
-steps:
-  - label: ":eslint: JS Lint"
-    command:
-      - "yarn install"
-      - "yarn lint:js"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: ":tslint: TS Lint"
-    command:
-      - "yarn install"
-      - "yarn lint:ts"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: ":typescript: Types Lint"
-    command:
-      - "yarn install"
-      - "yarn lint:types"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: "🛠 Build"
-    command:
-      - "yarn install"
-      - "yarn build"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: ":jest: Tests"
-    command:
-      - "yarn install"
-      - "yarn test"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - label: "📃 Docs"
-    command:
-      - "yarn install"
-      - "yarn gendoc"
-    plugins:
-      - docker#v3.0.1:
-          image: "node:12"
-
-  - wait
-
-  - label: "🐴 Trigger matrix-react-sdk"
-    trigger: "matrix-react-sdk"
-    branches: "develop"
-    build:
-        branch: "develop"
-        message: "[js-sdk] ${BUILDKITE_MESSAGE}"
-    async: true

CHANGELOG.md

@@ -1,3 +1,161 @@
+Changes in [5.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.2.0) (2020-03-30)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0-rc.1...v5.2.0)
+
+ * Fix isVerified returning false
+   [\#1290](https://github.com/matrix-org/matrix-js-sdk/pull/1290)
+
+Changes in [5.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.2.0-rc.1) (2020-03-26)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.1...v5.2.0-rc.1)
+
+ * Add a flag for whether cross signing signatures are trusted
+   [\#1285](https://github.com/matrix-org/matrix-js-sdk/pull/1285)
+ * Cache user and self signing keys during bootstrap
+   [\#1282](https://github.com/matrix-org/matrix-js-sdk/pull/1282)
+ * remove unnecessary promise
+   [\#1283](https://github.com/matrix-org/matrix-js-sdk/pull/1283)
+ * Functions to cache session backups key automatically
+   [\#1281](https://github.com/matrix-org/matrix-js-sdk/pull/1281)
+ * Add function for checking cross-signing is ready
+   [\#1279](https://github.com/matrix-org/matrix-js-sdk/pull/1279)
+ * Use symmetric encryption for SSSS
+   [\#1228](https://github.com/matrix-org/matrix-js-sdk/pull/1228)
+ * Migrate SSSS to use symmetric algorithm
+   [\#1238](https://github.com/matrix-org/matrix-js-sdk/pull/1238)
+ * Migration to symmetric SSSS
+   [\#1272](https://github.com/matrix-org/matrix-js-sdk/pull/1272)
+ * Reduce number of one-time-key requests
+   [\#1280](https://github.com/matrix-org/matrix-js-sdk/pull/1280)
+ * Fix: assume the requested method is supported by other party with to_device
+   [\#1275](https://github.com/matrix-org/matrix-js-sdk/pull/1275)
+ * Use checkDeviceTrust when computing untrusted devices
+   [\#1278](https://github.com/matrix-org/matrix-js-sdk/pull/1278)
+ * Add a store for backup keys
+   [\#1271](https://github.com/matrix-org/matrix-js-sdk/pull/1271)
+ * Upload only new device signature of master key
+   [\#1268](https://github.com/matrix-org/matrix-js-sdk/pull/1268)
+ * Expose prepareToEncrypt in the client API
+   [\#1270](https://github.com/matrix-org/matrix-js-sdk/pull/1270)
+ * Don't kill the whole device download if one device gives an error
+   [\#1269](https://github.com/matrix-org/matrix-js-sdk/pull/1269)
+ * Handle racing .start event during self verification
+   [\#1267](https://github.com/matrix-org/matrix-js-sdk/pull/1267)
+ * A crypto.keySignatureUploadFailure event reported the wrong source
+   [\#1266](https://github.com/matrix-org/matrix-js-sdk/pull/1266)
+ * Fix editing of unsent messages by waiting for actual event id
+   [\#1263](https://github.com/matrix-org/matrix-js-sdk/pull/1263)
+ * Fix: ensureOlmSessionsForDevices parameter format
+   [\#1264](https://github.com/matrix-org/matrix-js-sdk/pull/1264)
+ * Remove stuff that yarn install doesn't think we need
+   [\#1261](https://github.com/matrix-org/matrix-js-sdk/pull/1261)
+ * Fix: prevent error being thrown during sync in some cases
+   [\#1258](https://github.com/matrix-org/matrix-js-sdk/pull/1258)
+ * Force `is_verified` for key backups to bool and fix computation
+   [\#1259](https://github.com/matrix-org/matrix-js-sdk/pull/1259)
+ * Add a method for legacy single device verification, returning a verification
+   request
+   [\#1257](https://github.com/matrix-org/matrix-js-sdk/pull/1257)
+ * yarn upgrade
+   [\#1256](https://github.com/matrix-org/matrix-js-sdk/pull/1256)
+
+Changes in [5.1.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.1) (2020-03-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.1-rc.1...v5.1.1)
+
+ * Fix: ensureOlmSessionsForDevices parameter format
+   [\#1265](https://github.com/matrix-org/matrix-js-sdk/pull/1265)
+ * Fix: prevent error being thrown during sync in some cases
+   [\#1262](https://github.com/matrix-org/matrix-js-sdk/pull/1262)
+ * Force `is_verified` for key backups to bool and fix computation
+   [\#1260](https://github.com/matrix-org/matrix-js-sdk/pull/1260)
+
+Changes in [5.1.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.1-rc.1) (2020-03-11)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.0...v5.1.1-rc.1)
+
+ * refactor megolm encryption to improve perceived speed
+   [\#1252](https://github.com/matrix-org/matrix-js-sdk/pull/1252)
+ * Remove v1 identity server fallbacks
+   [\#1253](https://github.com/matrix-org/matrix-js-sdk/pull/1253)
+ * Use alt_aliases instead of local ones for room names
+   [\#1251](https://github.com/matrix-org/matrix-js-sdk/pull/1251)
+ * Upload cross-signing key signatures in the background
+   [\#1250](https://github.com/matrix-org/matrix-js-sdk/pull/1250)
+ * Fix secret sharing names to match spec
+   [\#1249](https://github.com/matrix-org/matrix-js-sdk/pull/1249)
+ * Cleanup: remove crypto.verification.start event
+   [\#1248](https://github.com/matrix-org/matrix-js-sdk/pull/1248)
+ * Fix regression in key backup request params
+   [\#1246](https://github.com/matrix-org/matrix-js-sdk/pull/1246)
+ * Use cross-signing trust to mark backups verified
+   [\#1244](https://github.com/matrix-org/matrix-js-sdk/pull/1244)
+ * Check both cross-signing and local trust for key sharing
+   [\#1243](https://github.com/matrix-org/matrix-js-sdk/pull/1243)
+ * Fixed up tests to match new way that crypto stores are created
+   [\#1242](https://github.com/matrix-org/matrix-js-sdk/pull/1242)
+ * Store USK and SSK locally
+   [\#1235](https://github.com/matrix-org/matrix-js-sdk/pull/1235)
+ * Use unpadded base64 for QR code secrets
+   [\#1236](https://github.com/matrix-org/matrix-js-sdk/pull/1236)
+ * Don't require .done event for finishing self-verification
+   [\#1239](https://github.com/matrix-org/matrix-js-sdk/pull/1239)
+ * Don't cancel as 3rd party in verification request
+   [\#1237](https://github.com/matrix-org/matrix-js-sdk/pull/1237)
+ * Verification: log when switching start event
+   [\#1234](https://github.com/matrix-org/matrix-js-sdk/pull/1234)
+ * Perform crypto store operations directly after transaction
+   [\#1233](https://github.com/matrix-org/matrix-js-sdk/pull/1233)
+ * More verification request logging
+   [\#1232](https://github.com/matrix-org/matrix-js-sdk/pull/1232)
+ * Upgrade deps
+   [\#1231](https://github.com/matrix-org/matrix-js-sdk/pull/1231)
+
+Changes in [5.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.0) (2020-03-02)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.1.0-rc.1...v5.1.0)
+
+ * No changes since rc.1
+
+Changes in [5.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.1.0-rc.1) (2020-02-26)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.0.1...v5.1.0-rc.1)
+
+ * Add latest dist-tag for releases
+   [\#1230](https://github.com/matrix-org/matrix-js-sdk/pull/1230)
+ * Add room method for alt_aliases
+   [\#1225](https://github.com/matrix-org/matrix-js-sdk/pull/1225)
+ * Remove buildkite pipeline
+   [\#1227](https://github.com/matrix-org/matrix-js-sdk/pull/1227)
+ * don't assume verify has been called when receiving a cancellation in
+   verifier
+   [\#1226](https://github.com/matrix-org/matrix-js-sdk/pull/1226)
+ * Reduce secret size for new binary packing
+   [\#1221](https://github.com/matrix-org/matrix-js-sdk/pull/1221)
+ * misc rageshake fixes
+   [\#1223](https://github.com/matrix-org/matrix-js-sdk/pull/1223)
+ * Fix cancelled historical requests not appearing as cancelled
+   [\#1220](https://github.com/matrix-org/matrix-js-sdk/pull/1220)
+ * Fix renaming error that broke QR code verification
+   [\#1217](https://github.com/matrix-org/matrix-js-sdk/pull/1217)
+
+Changes in [5.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.0.1) (2020-02-19)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.0.0...v5.0.1)
+
+ * add method for new /aliases endpoint
+   [\#1219](https://github.com/matrix-org/matrix-js-sdk/pull/1219)
+ * method for checking if other party supports verification method
+   [\#1213](https://github.com/matrix-org/matrix-js-sdk/pull/1213)
+ * add local echo state for accepting or declining a verif req
+   [\#1210](https://github.com/matrix-org/matrix-js-sdk/pull/1210)
+ * make logging compatible with rageshakes
+   [\#1214](https://github.com/matrix-org/matrix-js-sdk/pull/1214)
+ * Find existing requests when starting a new verification request
+   [\#1209](https://github.com/matrix-org/matrix-js-sdk/pull/1209)
+ * log MAC calculation during SAS
+   [\#1211](https://github.com/matrix-org/matrix-js-sdk/pull/1211)
+
 Changes in [5.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.0.0) (2020-02-17)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.0.0-rc.1...v5.0.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "matrix-js-sdk",
-  "version": "5.0.0",
+  "version": "5.2.0",
   "description": "Matrix Client-Server SDK for Javascript",
   "scripts": {
     "prepare": "yarn build",
@@ -78,7 +78,9 @@
     "eslint-plugin-babel": "^5.3.0",
     "eslint-plugin-jest": "^23.0.4",
     "exorcist": "^1.0.1",
+    "fake-indexeddb": "^3.0.0",
     "jest": "^24.9.0",
+    "jest-localstorage-mock": "^2.4.0",
     "jsdoc": "^3.5.5",
     "matrix-mock-request": "^1.2.3",
     "olm": "https://packages.matrix.org/npm/olm/olm-3.1.4.tgz",

release.sh

@@ -294,14 +294,16 @@ fi
 rm "${release_text}"
 rm "${latest_changes}"
 
-npm_publish_flags=''
-if [ $prerelease -eq 1 ]; then
-    # Tag prereleases as `next` so the last stable release remains the default
-    npm_publish_flags='--tag next'
-fi
 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-npm publish $npm_publish_flags
+# Tag both releases and prereleases as `next` so the last stable release remains
+# the default.
+npm publish --tag next
+if [ $prerelease -eq 0 ]; then
+    # For a release, also add the default `latest` tag.
+    package=$(cat package.json | jq -er .name)
+    npm dist-tag add "$package@$release" latest
+fi
 
 if [ -z "$skip_jsdoc" ]; then
     echo "generating jsdocs"

spec/unit/crypto/CrossSigningInfo.spec.js

@@ -0,0 +1,248 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import '../../olm-loader';
+import {
+    CrossSigningInfo,
+    createCryptoStoreCacheCallbacks,
+} from '../../../src/crypto/CrossSigning';
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+const userId = "@alice:example.com";
+
+// Private key for tests only
+const testKey = new Uint8Array([
+    0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+    0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+    0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+    0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+]);
+
+const types = [
+    { type: "master", shouldCache: false },
+    { type: "self_signing", shouldCache: true },
+    { type: "user_signing", shouldCache: true },
+    { type: "invalid", shouldCache: false },
+];
+
+const badKey = Uint8Array.from(testKey);
+badKey[0] ^= 1;
+
+const masterKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
+
+describe("CrossSigningInfo.getCrossSigningKey", function() {
+    if (!global.Olm) {
+        console.warn('Not running megolm backup unit tests: libolm not present');
+        return;
+    }
+
+    beforeAll(function() {
+        return global.Olm.init();
+    });
+
+    it("should throw if no callback is provided", async () => {
+        const info = new CrossSigningInfo(userId);
+        await expect(info.getCrossSigningKey("master")).rejects.toThrow();
+    });
+
+    it.each(types)("should throw if the callback returns falsey",
+                   async ({type, shouldCache}) => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => false,
+        });
+        await expect(info.getCrossSigningKey(type)).rejects.toThrow("falsey");
+    });
+
+    it("should throw if the expected key doesn't come back", async () => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => masterKeyPub,
+        });
+        await expect(info.getCrossSigningKey("master", "")).rejects.toThrow();
+    });
+
+    it("should return a key from its callback", async () => {
+        const info = new CrossSigningInfo(userId, {
+            getCrossSigningKey: () => testKey,
+        });
+        const [pubKey, ab] = await info.getCrossSigningKey("master", masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(ab).toEqual({a: 106712, b: 106712});
+    });
+
+    it.each(types)("should request a key from the cache callback (if set)" +
+                   " and does not call app if one is found" +
+                   " %o",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockImplementation(() => {
+            if (shouldCache) {
+                return Promise.reject(new Error("Regular callback called"));
+            } else {
+                return Promise.resolve(testKey);
+            }
+        });
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+        if (shouldCache) {
+            expect(getCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+        }
+    });
+
+    it.each(types)("should store a key with the cache callback (if set)",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(shouldCache ? 1 : 0);
+        if (shouldCache) {
+            expect(storeCrossSigningKeyCache.mock.calls[0][0]).toBe(type);
+            expect(storeCrossSigningKeyCache.mock.calls[0][1]).toBe(testKey);
+        }
+    });
+
+    it.each(types)("does not store a bad key to the cache",
+                   async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(badKey);
+        const storeCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { storeCrossSigningKeyCache },
+        );
+        await expect(info.getCrossSigningKey(type, masterKeyPub)).rejects.toThrow();
+        expect(storeCrossSigningKeyCache.mock.calls.length).toEqual(0);
+    });
+
+    it.each(types)("does not store a value to the cache if it came from the cache",
+        async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockImplementation(() => {
+            if (shouldCache) {
+                return Promise.reject(new Error("Regular callback called"));
+            } else {
+                return Promise.resolve(testKey);
+            }
+        });
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(testKey);
+        const storeCrossSigningKeyCache = jest.fn().mockRejectedValue(
+            new Error("Tried to store a value from cache"),
+        );
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(0);
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+    });
+
+    it.each(types)("requests a key from the cache callback (if set) and then calls app" +
+        " if one is not found", async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(undefined);
+        const storeCrossSigningKeyCache = jest.fn();
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKey.mock.calls.length).toBe(1);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+
+        /* Also expect that the cache gets updated */
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+    });
+
+    it.each(types)("requests a key from the cache callback (if set) and then" +
+        " calls app if that key doesn't match", async ({ type, shouldCache }) => {
+        const getCrossSigningKey = jest.fn().mockResolvedValue(testKey);
+        const getCrossSigningKeyCache = jest.fn().mockResolvedValue(badKey);
+        const storeCrossSigningKeyCache = jest.fn();
+        const info = new CrossSigningInfo(
+            userId,
+            { getCrossSigningKey },
+            { getCrossSigningKeyCache, storeCrossSigningKeyCache },
+        );
+        const [pubKey] = await info.getCrossSigningKey(type, masterKeyPub);
+        expect(pubKey).toEqual(masterKeyPub);
+        expect(getCrossSigningKey.mock.calls.length).toBe(1);
+        expect(getCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+
+        /* Also expect that the cache gets updated */
+        expect(storeCrossSigningKeyCache.mock.calls.length).toBe(shouldCache ? 1 : 0);
+    });
+});
+
+/*
+ * Note that MemoryStore is weird.  It's only used for testing - as far as I can tell,
+ * it's not possible to get one in normal execution unless you hack as we do here.
+ */
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("CrossSigning > createCryptoStoreCacheCallbacks [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(() => {
+        store = dbFactory();
+    });
+
+    beforeEach(async () => {
+        await store.deleteAllData();
+    });
+
+    it("should cache data to the store and retrieve it", async () => {
+        await store.startup();
+        const { getCrossSigningKeyCache, storeCrossSigningKeyCache } =
+          createCryptoStoreCacheCallbacks(store);
+        await storeCrossSigningKeyCache("self_signing", testKey);
+
+        // If we've not saved anything, don't expect anything
+        // Definitely don't accidentally return the wrong key for the type
+        const nokey = await getCrossSigningKeyCache("self", "");
+        expect(nokey).toBeNull();
+
+        const key = await getCrossSigningKeyCache("self_signing", "");
+        expect(key).toEqual(testKey);
+    });
+});

spec/unit/crypto/algorithms/megolm.spec.js

@@ -297,6 +297,10 @@ describe("MegolmDecryption", function() {
                 },
             }));
 
+            mockCrypto.checkDeviceTrust.mockReturnValue({
+                isVerified: () => false,
+            });
+
             const megolmEncryption = new MegolmEncryption({
                 userId: '@user:id',
                 crypto: mockCrypto,
@@ -320,14 +324,14 @@ describe("MegolmDecryption", function() {
 
             // this should have claimed a key for alice as it's starting a new session
             expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
-                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
             );
             expect(mockCrypto.downloadKeys).toHaveBeenCalledWith(
                 ['@alice:home.server'], false,
             );
             expect(mockBaseApis.sendToDevice).toHaveBeenCalled();
             expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith(
-                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519',
+                [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000,
             );
 
             mockBaseApis.claimOneTimeKeys.mockReset();
@@ -516,21 +520,22 @@ describe("MegolmDecryption", function() {
             };
         };
 
-        let run = false;
-        aliceClient.sendToDevice = async (msgtype, contentMap) => {
-            run = true;
-            expect(msgtype).toBe("org.matrix.room_key.withheld");
-            expect(contentMap).toStrictEqual({
-                '@bob:example.com': {
-                    bobdevice: {
-                        algorithm: "m.megolm.v1.aes-sha2",
-                        code: 'm.no_olm',
-                        reason: 'Unable to establish a secure channel.',
-                        sender_key: aliceDevice.deviceCurve25519Key,
+        const sendPromise = new Promise((resolve, reject) => {
+            aliceClient.sendToDevice = async (msgtype, contentMap) => {
+                expect(msgtype).toBe("org.matrix.room_key.withheld");
+                expect(contentMap).toStrictEqual({
+                    '@bob:example.com': {
+                        bobdevice: {
+                            algorithm: "m.megolm.v1.aes-sha2",
+                            code: 'm.no_olm',
+                            reason: 'Unable to establish a secure channel.',
+                            sender_key: aliceDevice.deviceCurve25519Key,
+                        },
                     },
-                },
-            });
-        };
+                });
+                resolve();
+            };
+        });
 
         const event = new MatrixEvent({
             type: "m.room.message",
@@ -540,8 +545,7 @@ describe("MegolmDecryption", function() {
             content: {},
         });
         await aliceClient._crypto.encryptEvent(event, aliceRoom);
-
-        expect(run).toBe(true);
+        await sendPromise;
     });
 
     it("throws an error describing why it doesn't have a key", async function() {
@@ -598,6 +602,7 @@ describe("MegolmDecryption", function() {
             aliceClient.initCrypto(),
             bobClient.initCrypto(),
         ]);
+        aliceClient._crypto.downloadKeys = async () => {};
         const bobDevice = bobClient._crypto._olmDevice;
 
         const roomId = "!someroom";
@@ -649,6 +654,7 @@ describe("MegolmDecryption", function() {
             bobClient.initCrypto(),
         ]);
         const bobDevice = bobClient._crypto._olmDevice;
+        aliceClient._crypto.downloadKeys = async () => {};
 
         const roomId = "!someroom";
 

spec/unit/crypto/backup.spec.js

@@ -541,5 +541,31 @@ describe("MegolmBackup", function() {
                 expect(res.clearEvent.content).toEqual('testytest');
             });
         });
+
+        it('has working cache functions', async function() {
+            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
+            await client._crypto.storeSessionBackupPrivateKey(key);
+            const result = await client._crypto.getSessionBackupPrivateKey();
+            expect(result).toEqual(key);
+        });
+
+        it('caches session backup keys as it encounters them', async function() {
+            const cachedNull = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedNull).toBeNull();
+            client._http.authedRequest = function() {
+                return Promise.resolve(KEY_BACKUP_DATA);
+            };
+            await new Promise((resolve) => {
+                client.restoreKeyBackupWithRecoveryKey(
+                    "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d",
+                    ROOM_ID,
+                    SESSION_ID,
+                    BACKUP_INFO,
+                    { cacheCompleteCallback: resolve },
+                );
+            });
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            expect(cachedKey).not.toBeNull();
+        });
     });
 });

spec/unit/crypto/secrets.spec.js

@@ -16,11 +16,20 @@ limitations under the License.
 
 import '../../olm-loader';
 import * as olmlib from "../../../src/crypto/olmlib";
-import {SECRET_STORAGE_ALGORITHM_V1} from "../../../src/crypto/SecretStorage";
+import {SECRET_STORAGE_ALGORITHM_V1_AES} from "../../../src/crypto/SecretStorage";
 import {MatrixEvent} from "../../../src/models/event";
 import {TestClient} from '../../TestClient';
 import {makeTestClients} from './verification/util';
 
+import * as utils from "../../../src/utils";
+
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    console.log('nodejs was compiled without crypto support');
+}
+
 async function makeTestClient(userInfo, options) {
     const client = (new TestClient(
         userInfo.userId, userInfo.deviceId, undefined, undefined, options,
@@ -51,9 +60,8 @@ describe("Secrets", function() {
     });
 
     it("should store and retrieve a secret", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const pubkey = decryption.generate_key();
-        const privkey = decryption.get_private_key();
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
 
         const signing = new global.Olm.PkSigning();
         const signingKey = signing.generate_seed();
@@ -69,7 +77,7 @@ describe("Secrets", function() {
 
         const getKey = jest.fn(e => {
             expect(Object.keys(e.keys)).toEqual(["abc"]);
-            return ['abc', privkey];
+            return ['abc', key];
         });
 
         const alice = await makeTestClient(
@@ -100,8 +108,7 @@ describe("Secrets", function() {
         };
 
         const keyAccountData = {
-            algorithm: SECRET_STORAGE_ALGORITHM_V1,
-            pubkey: pubkey,
+            algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
         };
         await alice._crypto._crossSigningInfo.signObject(keyAccountData, 'master');
 
@@ -112,11 +119,11 @@ describe("Secrets", function() {
             }),
         ]);
 
-        expect(await secretStorage.isStored("foo")).toBe(false);
+        expect(await secretStorage.isStored("foo")).toBeFalsy();
 
         await secretStorage.store("foo", "bar", ["abc"]);
 
-        expect(await secretStorage.isStored("foo")).toBe(true);
+        expect(await secretStorage.isStored("foo")).toBeTruthy();
         expect(await secretStorage.get("foo")).toBe("bar");
 
         expect(getKey).toHaveBeenCalled();
@@ -149,6 +156,13 @@ describe("Secrets", function() {
     });
 
     it("should encrypt with default key if keys is null", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+        const getKey = jest.fn(e => {
+            expect(Object.keys(e.keys)).toEqual([newKeyId]);
+            return [newKeyId, key];
+        });
+
         let keys = {};
         const alice = await makeTestClient(
             {userId: "@alice:example.com", deviceId: "Osborne2"},
@@ -156,6 +170,7 @@ describe("Secrets", function() {
                 cryptoCallbacks: {
                     getCrossSigningKey: t => keys[t],
                     saveCrossSigningKeys: k => keys = k,
+                    getSecretStorageKey: getKey,
                 },
             },
         );
@@ -170,7 +185,7 @@ describe("Secrets", function() {
         alice.resetCrossSigningKeys();
 
         const newKeyId = await alice.addSecretStorageKey(
-            SECRET_STORAGE_ALGORITHM_V1,
+            SECRET_STORAGE_ALGORITHM_V1_AES,
         );
         // we don't await on this because it waits for the event to come down the sync
         // which won't happen in the test setup
@@ -252,11 +267,22 @@ describe("Secrets", function() {
     });
 
     it("bootstraps when no storage or cross-signing keys locally", async function() {
+        const key = new Uint8Array(16);
+        for (let i = 0; i < 16; i++) key[i] = i;
+        const getKey = jest.fn(e => {
+            return [Object.keys(e.keys)[0], key];
+        });
+
         const bob = await makeTestClient(
             {
                 userId: "@bob:example.com",
                 deviceId: "bob1",
             },
+            {
+                cryptoCallbacks: {
+                    getSecretStorageKey: getKey,
+                },
+            },
         );
         bob.uploadDeviceSigningKeys = async () => {};
         bob.uploadKeySignatures = async () => {};

spec/unit/crypto/verification/sas.spec.js

@@ -122,8 +122,8 @@ describe("SAS verification", function() {
             bobSasEvent = null;
 
             bobPromise = new Promise((resolve, reject) => {
-                bob.client.on("crypto.verification.start", (verifier) => {
-                    verifier.on("show_sas", (e) => {
+                bob.client.on("crypto.verification.request", request => {
+                    request.verifier.on("show_sas", (e) => {
                         if (!e.sas.emoji || !e.sas.decimal) {
                             e.cancel();
                         } else if (!aliceSasEvent) {
@@ -139,7 +139,7 @@ describe("SAS verification", function() {
                             }
                         }
                     });
-                    resolve(verifier);
+                    resolve(request.verifier);
                 });
             });
 
@@ -339,11 +339,11 @@ describe("SAS verification", function() {
         };
 
         const bobPromise = new Promise((resolve, reject) => {
-            bob.client.on("crypto.verification.start", (verifier) => {
-                verifier.on("show_sas", (e) => {
+            bob.client.on("crypto.verification.request", request => {
+                request.verifier.on("show_sas", (e) => {
                     e.mismatch();
                 });
-                resolve(verifier);
+                resolve(request.verifier);
             });
         });
 

spec/unit/crypto/verification/secret_request.spec.js

@@ -0,0 +1,104 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {VerificationBase} from '../../../../src/crypto/verification/Base';
+import {CrossSigningInfo} from '../../../../src/crypto/CrossSigning';
+import {encodeBase64} from "../../../../src/crypto/olmlib";
+import {setupWebcrypto, teardownWebcrypto} from './util';
+
+jest.useFakeTimers();
+
+// Private key for tests only
+const testKey = new Uint8Array([
+    0xda, 0x5a, 0x27, 0x60, 0xe3, 0x3a, 0xc5, 0x82,
+    0x9d, 0x12, 0xc3, 0xbe, 0xe8, 0xaa, 0xc2, 0xef,
+    0xae, 0xb1, 0x05, 0xc1, 0xe7, 0x62, 0x78, 0xa6,
+    0xd7, 0x1f, 0xf8, 0x2c, 0x51, 0x85, 0xf0, 0x1d,
+]);
+const testKeyPub = "nqOvzeuGWT/sRx3h7+MHoInYj3Uk2LD/unI9kDYcHwk";
+
+describe("self-verifications", () => {
+    beforeAll(function() {
+        setupWebcrypto();
+        return global.Olm.init();
+    });
+
+    afterAll(() => {
+        teardownWebcrypto();
+    });
+
+    it("triggers a request for key sharing upon completion", async () => {
+        const userId = "@test:localhost";
+
+        const cacheCallbacks = {
+            getCrossSigningKeyCache: jest.fn().mockReturnValue(null),
+            storeCrossSigningKeyCache: jest.fn(),
+        };
+
+        const _crossSigningInfo = new CrossSigningInfo(
+            userId,
+            {},
+            cacheCallbacks,
+        );
+        _crossSigningInfo.keys = {
+            self_signing: { keys: { X: testKeyPub } },
+            user_signing: { keys: { X: testKeyPub } },
+        };
+
+        const _secretStorage = {
+            request: jest.fn().mockReturnValue({
+                promise: Promise.resolve(encodeBase64(testKey)),
+            }),
+        };
+
+        const client = {
+            _crypto: {
+                _crossSigningInfo,
+                _secretStorage,
+            },
+            getUserId: () => userId,
+        };
+
+        const request = {
+            onVerifierFinished: () => undefined,
+        };
+
+        const verification = new VerificationBase(
+            undefined, // channel
+            client, // baseApis
+            userId,
+            "ABC", // deviceId
+            undefined, // startEvent
+            request,
+        );
+        verification._resolve = () => undefined;
+
+        const result = await verification.done();
+
+        /* We should request, and store, two keys */
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(2);
+        expect(_secretStorage.request.mock.calls.length).toBe(2);
+
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
+          .toEqual(testKey);
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[1][1])
+          .toEqual(testKey);
+
+        expect(result).toBeInstanceOf(Array);
+        expect(result[0][0]).toBe(testKeyPub);
+        expect(result[1][0]).toBe(testKeyPub);
+    });
+});

spec/unit/crypto/verification/verification_request.spec.js

@@ -120,7 +120,6 @@ async function distributeEvent(ownRequest, theirRequest, event) {
 }
 
 describe("verification request unit tests", function() {
-
     beforeAll(function() {
         setupWebcrypto();
     });

spec/unit/room.spec.js

@@ -617,15 +617,10 @@ describe("Room", function() {
                 }, event: true,
             })]);
         };
-        const setAliases = function(aliases, stateKey) {
-            if (!stateKey) {
-                stateKey = aliases.length
-                    ? aliases[0].split(':').splice(1).join(':') // domain+port
-                    : 'fibble';
-            }
+        const setAltAliases = function(aliases) {
             room.addLiveEvents([utils.mkEvent({
-                type: "m.room.aliases", room: roomId, skey: stateKey, content: {
-                    aliases: aliases,
+                type: "m.room.canonical_alias", room: roomId, skey: "", content: {
+                    alt_aliases: aliases,
                 }, event: true,
             })]);
         };
@@ -862,7 +857,7 @@ describe("Room", function() {
             "(invite join_rules) rooms if a room name doesn't exist.", function() {
                 const alias = "#room_alias:here";
                 setJoinRule("invite");
-                setAliases([alias, "#another:here"]);
+                setAltAliases([alias, "#another:here"]);
                 room.recalculate();
                 const name = room.name;
                 expect(name).toEqual(alias);
@@ -872,7 +867,7 @@ describe("Room", function() {
             "(public join_rules) rooms if a room name doesn't exist.", function() {
                 const alias = "#room_alias:here";
                 setJoinRule("public");
-                setAliases([alias, "#another:here"]);
+                setAltAliases([alias, "#another:here"]);
                 room.recalculate();
                 const name = room.name;
                 expect(name).toEqual(alias);

src/base-apis.js

@@ -28,13 +28,7 @@ import {SERVICE_TYPES} from './service-types';
 import {logger} from './logger';
 import {PushProcessor} from "./pushprocessor";
 import * as utils from "./utils";
-import {
-    MatrixHttpApi,
-    PREFIX_IDENTITY_V1,
-    PREFIX_IDENTITY_V2,
-    PREFIX_R0,
-    PREFIX_UNSTABLE,
-} from "./http-api";
+import {MatrixHttpApi, PREFIX_IDENTITY_V2, PREFIX_R0, PREFIX_UNSTABLE} from "./http-api";
 
 function termsUrlForService(serviceType, baseUrl) {
     switch (serviceType) {
@@ -1119,6 +1113,21 @@ MatrixBaseApis.prototype.deleteAlias = function(alias, callback) {
     );
 };
 
+/**
+ * @param {string} roomId
+ * @param {module:client.callback} callback Optional.
+ * @return {Promise} Resolves: an object with an `aliases` property, containing an array of local aliases
+ * @return {module:http-api.MatrixError} Rejects: with an error response.
+ */
+MatrixBaseApis.prototype.unstableGetLocalAliases =
+function(roomId, callback) {
+    const path = utils.encodeUri("/rooms/$roomId/aliases",
+        {$roomId: roomId});
+    const prefix = PREFIX_UNSTABLE + "/org.matrix.msc2432";
+    return this._http.authedRequest(callback, "GET", path,
+        null, null, { prefix });
+};
+
 /**
  * Get room info for the given alias.
  * @param {string} alias The room alias to resolve.
@@ -1761,10 +1770,13 @@ MatrixBaseApis.prototype.downloadKeysForUsers = function(userIds, opts) {
  *
  * @param {string} [key_algorithm = signed_curve25519]  desired key type
  *
+ * @param {number} [timeout] the time (in milliseconds) to wait for keys from remote
+ *     servers
+ *
  * @return {Promise} Resolves: result object. Rejects: with
  *     an error response ({@link module:http-api.MatrixError}).
  */
-MatrixBaseApis.prototype.claimOneTimeKeys = function(devices, key_algorithm) {
+MatrixBaseApis.prototype.claimOneTimeKeys = function(devices, key_algorithm, timeout) {
     const queries = {};
 
     if (key_algorithm === undefined) {
@@ -1779,6 +1791,9 @@ MatrixBaseApis.prototype.claimOneTimeKeys = function(devices, key_algorithm) {
         query[deviceId] = key_algorithm;
     }
     const content = {one_time_keys: queries};
+    if (timeout) {
+        content.timeout = timeout;
+    }
     const path = "/keys/claim";
     return this._http.authedRequest(undefined, "POST", path, undefined, content);
 };
@@ -1880,28 +1895,10 @@ MatrixBaseApis.prototype.requestEmailToken = async function(
         next_link: nextLink,
     };
 
-    try {
-        const response = await this._http.idServerRequest(
-            undefined, "POST", "/validate/email/requestToken",
-            params, PREFIX_IDENTITY_V2, identityAccessToken,
-        );
-        // TODO: Fold callback into above call once v1 path below is removed
-        if (callback) callback(null, response);
-        return response;
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                callback, "POST", "/validate/email/requestToken",
-                params, PREFIX_IDENTITY_V1,
-            );
-        }
-        if (callback) callback(err);
-        throw err;
-    }
+    return await this._http.idServerRequest(
+        callback, "POST", "/validate/email/requestToken",
+        params, PREFIX_IDENTITY_V2, identityAccessToken,
+    );
 };
 
 /**
@@ -1948,28 +1945,10 @@ MatrixBaseApis.prototype.requestMsisdnToken = async function(
         next_link: nextLink,
     };
 
-    try {
-        const response = await this._http.idServerRequest(
-            undefined, "POST", "/validate/msisdn/requestToken",
-            params, PREFIX_IDENTITY_V2, identityAccessToken,
-        );
-        // TODO: Fold callback into above call once v1 path below is removed
-        if (callback) callback(null, response);
-        return response;
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                callback, "POST", "/validate/msisdn/requestToken",
-                params, PREFIX_IDENTITY_V1,
-            );
-        }
-        if (callback) callback(err);
-        throw err;
-    }
+    return await this._http.idServerRequest(
+        callback, "POST", "/validate/msisdn/requestToken",
+        params, PREFIX_IDENTITY_V2, identityAccessToken,
+    );
 };
 
 /**
@@ -2003,24 +1982,10 @@ MatrixBaseApis.prototype.submitMsisdnToken = async function(
         token: msisdnToken,
     };
 
-    try {
-        return await this._http.idServerRequest(
-            undefined, "POST", "/validate/msisdn/submitToken",
-            params, PREFIX_IDENTITY_V2, identityAccessToken,
-        );
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                undefined, "POST", "/validate/msisdn/submitToken",
-                params, PREFIX_IDENTITY_V1,
-            );
-        }
-        throw err;
-    }
+    return await this._http.idServerRequest(
+        undefined, "POST", "/validate/msisdn/submitToken",
+        params, PREFIX_IDENTITY_V2, identityAccessToken,
+    );
 };
 
 /**
@@ -2175,53 +2140,32 @@ MatrixBaseApis.prototype.lookupThreePid = async function(
     callback,
     identityAccessToken,
 ) {
-    try {
-        // Note: we're using the V2 API by calling this function, but our
-        // function contract requires a V1 response. We therefore have to
-        // convert it manually.
-        const response = await this.identityHashedLookup(
-            [[address, medium]], identityAccessToken,
-        );
-        const result = response.find(p => p.address === address);
-        if (!result) {
-            // TODO: Fold callback into above call once v1 path below is removed
-            if (callback) callback(null, {});
-            return {};
-        }
-
-        const mapping = {
-            address,
-            medium,
-            mxid: result.mxid,
-
-            // We can't reasonably fill these parameters:
-            // not_before
-            // not_after
-            // ts
-            // signatures
-        };
-
-        // TODO: Fold callback into above call once v1 path below is removed
-        if (callback) callback(null, mapping);
-        return mapping;
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            const params = {
-                medium: medium,
-                address: address,
-            };
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                callback, "GET", "/lookup",
-                params, PREFIX_IDENTITY_V1,
-            );
-        }
-        if (callback) callback(err, undefined);
-        throw err;
+    // Note: we're using the V2 API by calling this function, but our
+    // function contract requires a V1 response. We therefore have to
+    // convert it manually.
+    const response = await this.identityHashedLookup(
+        [[address, medium]], identityAccessToken,
+    );
+    const result = response.find(p => p.address === address);
+    if (!result) {
+        if (callback) callback(null, {});
+        return {};
     }
+
+    const mapping = {
+        address,
+        medium,
+        mxid: result.mxid,
+
+        // We can't reasonably fill these parameters:
+        // not_before
+        // not_after
+        // ts
+        // signatures
+    };
+
+    if (callback) callback(null, mapping);
+    return mapping;
 };
 
 /**
@@ -2239,46 +2183,29 @@ MatrixBaseApis.prototype.bulkLookupThreePids = async function(
     query,
     identityAccessToken,
 ) {
-    try {
-        // Note: we're using the V2 API by calling this function, but our
-        // function contract requires a V1 response. We therefore have to
-        // convert it manually.
-        const response = await this.identityHashedLookup(
-            // We have to reverse the query order to get [address, medium] pairs
-            query.map(p => [p[1], p[0]]), identityAccessToken,
-        );
+    // Note: we're using the V2 API by calling this function, but our
+    // function contract requires a V1 response. We therefore have to
+    // convert it manually.
+    const response = await this.identityHashedLookup(
+        // We have to reverse the query order to get [address, medium] pairs
+        query.map(p => [p[1], p[0]]), identityAccessToken,
+    );
 
-        const v1results = [];
-        for (const mapping of response) {
-            const originalQuery = query.find(p => p[1] === mapping.address);
-            if (!originalQuery) {
-                throw new Error("Identity sever returned unexpected results");
-            }
-
-            v1results.push([
-                originalQuery[0], // medium
-                mapping.address,
-                mapping.mxid,
-            ]);
+    const v1results = [];
+    for (const mapping of response) {
+        const originalQuery = query.find(p => p[1] === mapping.address);
+        if (!originalQuery) {
+            throw new Error("Identity sever returned unexpected results");
         }
 
-        return {threepids: v1results};
-    } catch (err) {
-        if (err.cors === "rejected" || err.httpStatus === 404) {
-            // Fall back to deprecated v1 API for now
-            // TODO: Remove this path once v2 is only supported version
-            // See https://github.com/vector-im/riot-web/issues/10443
-            const params = {
-                threepids: query,
-            };
-            logger.warn("IS doesn't support v2, falling back to deprecated v1");
-            return await this._http.idServerRequest(
-                undefined, "POST", "/bulk_lookup", params,
-                PREFIX_IDENTITY_V1, identityAccessToken,
-            );
-        }
-        throw err;
+        v1results.push([
+            originalQuery[0], // medium
+            mapping.address,
+            mapping.mxid,
+        ]);
     }
+
+    return {threepids: v1results};
 };
 
 /**

src/client.js

@@ -217,8 +217,10 @@ function keyFromRecoverySession(session, decryptionKey) {
  * Args:
  *   {object} keys Information about the keys:
  *       {
- *           <key name>: {
- *               pubkey: {UInt8Array}
+ *           keys: {
+ *               <key name>: {
+ *                   pubkey: {UInt8Array}
+ *               }, ...
  *           }
  *       }
  *   {string} name the name of the value we want to read out of SSSS, for UI purposes.
@@ -281,13 +283,19 @@ export function MatrixClient(opts) {
     this.scheduler = opts.scheduler;
     if (this.scheduler) {
         const self = this;
-        this.scheduler.setProcessFunction(function(eventToSend) {
+        this.scheduler.setProcessFunction(async function(eventToSend) {
             const room = self.getRoom(eventToSend.getRoomId());
             if (eventToSend.status !== EventStatus.SENDING) {
                 _updatePendingEventStatus(room, eventToSend,
                                           EventStatus.SENDING);
             }
-            return _sendEventHttpRequest(self, eventToSend);
+            const res = await _sendEventHttpRequest(self, eventToSend);
+            if (room) {
+                // ensure we update pending event before the next scheduler run so that any listeners to event id
+                // updates on the synchronous event emitter get a chance to run first.
+                room.updatePendingEvent(eventToSend, EventStatus.SENT, res.event_id);
+            }
+            return res;
         });
     }
     this.clientRunning = false;
@@ -407,9 +415,9 @@ export function MatrixClient(opts) {
                     break;
                 }
 
-                highlightCount += this.getPushActionsForEvent(
-                    event,
-                ).tweaks.highlight ? 1 : 0;
+                const pushActions = this.getPushActionsForEvent(event);
+                highlightCount += pushActions.tweaks &&
+                    pushActions.tweaks.highlight ? 1 : 0;
             }
 
             // Note: we don't need to handle 'total' notifications because the counts
@@ -685,6 +693,9 @@ MatrixClient.prototype.initCrypto = async function() {
         throw new Error(`Cannot enable encryption: no cryptoStore provided`);
     }
 
+    logger.log("Crypto: Starting up crypto store...");
+    await this._cryptoStore.startup();
+
     // initialise the list of encrypted rooms (whether or not crypto is enabled)
     logger.log("Crypto: initialising roomlist...");
     await this._roomList.init();
@@ -749,7 +760,6 @@ MatrixClient.prototype.isCryptoEnabled = function() {
     return this._crypto !== null;
 };
 
-
 /**
  * Get the Ed25519 key for this device
  *
@@ -763,6 +773,19 @@ MatrixClient.prototype.getDeviceEd25519Key = function() {
     return this._crypto.getDeviceEd25519Key();
 };
 
+/**
+ * Get the Curve25519 key for this device
+ *
+ * @return {?string} base64-encoded curve25519 key. Null if crypto is
+ *    disabled.
+ */
+MatrixClient.prototype.getDeviceCurve25519Key = function() {
+    if (!this._crypto) {
+        return null;
+    }
+    return this._crypto.getDeviceCurve25519Key();
+};
+
 /**
  * Upload the device keys to the homeserver.
  * @return {object} A promise that will resolve when the keys are uploaded.
@@ -918,6 +941,20 @@ MatrixClient.prototype.requestVerificationDM = function(userId, roomId) {
     return this._crypto.requestVerificationDM(userId, roomId);
 };
 
+/**
+ * Finds a DM verification request that is already in progress for the given room id
+ *
+ * @param {string} roomId the room to use for verification
+ *
+ * @returns {module:crypto/verification/request/VerificationRequest?} the VerificationRequest that is in progress, if any
+ */
+MatrixClient.prototype.findVerificationRequestDMInProgress = function(roomId) {
+    if (this._crypto === null) {
+        throw new Error("End-to-end encryption disabled");
+    }
+    return this._crypto.findVerificationRequestDMInProgress(roomId);
+};
+
 /**
  * Request a key verification from another user.
  *
@@ -1097,6 +1134,12 @@ function wrapCryptoFuncs(MatrixClient, names) {
  * @returns {boolean} true if the key matches, otherwise false
  */
 
+/**
+ * Perform any background tasks that can be done before a message is ready to
+ * send, in order to speed up sending of the message.
+ *
+ * @param {module:models/room} room the room the event is in
+ */
 wrapCryptoFuncs(MatrixClient, [
     "resetCrossSigningKeys",
     "getCrossSigningId",
@@ -1105,6 +1148,11 @@ wrapCryptoFuncs(MatrixClient, [
     "checkDeviceTrust",
     "checkOwnCrossSigningTrust",
     "checkCrossSigningPrivateKey",
+    "legacyDeviceVerification",
+    "prepareToEncrypt",
+    "isCrossSigningReady",
+    "getCryptoTrustCrossSignedDevices",
+    "setCryptoTrustCrossSignedDevices",
 ]);
 
 /**
@@ -1203,7 +1251,9 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {
  * @param {boolean} checkKey check if the secret is encrypted by a trusted
  *     key
  *
- * @return {boolean} whether or not the secret is stored
+ * @return {object?} map of key name to key info the secret is encrypted
+ *     with, or null if it is not present or not encrypted with a trusted
+ *     key
  */
 
 /**
@@ -1251,6 +1301,7 @@ wrapCryptoFuncs(MatrixClient, [
     "bootstrapSecretStorage",
     "addSecretStorageKey",
     "hasSecretStorageKey",
+    "secretStorageKeyNeedsUpgrade",
     "storeSecret",
     "getSecret",
     "isSecretStored",
@@ -1537,7 +1588,9 @@ MatrixClient.prototype.prepareKeyBackupVersion = async function(
 
 /**
  * Check whether the key backup private key is stored in secret storage.
- * @return {Promise<boolean>} Whether the backup key is stored.
+ * @return {Promise<object?>} map of key name to key info the secret is
+ *     encrypted with, or null if it is not present or not encrypted with a
+ *     trusted key
  */
 MatrixClient.prototype.isKeyBackupKeyStored = async function() {
     return this.isSecretStored("m.megolm_backup.v1", false /* checkKey */);
@@ -1690,6 +1743,35 @@ MatrixClient.prototype.isValidRecoveryKey = function(recoveryKey) {
     }
 };
 
+/**
+ * Get the raw key for a key backup from the password
+ * Used when migrating key backups into SSSS
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @param {string} password Passphrase
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @return {Promise<Buffer>} key backup key
+ */
+MatrixClient.prototype.keyBackupKeyFromPassword = function(
+    password, backupInfo,
+) {
+    return keyFromAuthData(backupInfo.auth_data, password);
+};
+
+/**
+ * Get the raw key for a key backup from the recovery key
+ * Used when migrating key backups into SSSS
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @param {string} recoveryKey The recovery key
+ * @return {Buffer} key backup key
+ */
+MatrixClient.prototype.keyBackupKeyFromRecoveryKey = function(recoveryKey) {
+    return decodeRecoveryKey(recoveryKey);
+};
+
 MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY = 'RESTORE_BACKUP_ERROR_BAD_KEY';
 
 /**
@@ -1701,15 +1783,16 @@ MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY = 'RESTORE_BACKUP_ERROR_BAD_KEY';
  * @param {string} [targetSessionId] Session ID to target a specific session.
  * Restores all sessions if omitted.
  * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
  * @return {Promise<object>} Status of restoration with `total` and `imported`
  * key counts.
  */
 MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
-    password, targetRoomId, targetSessionId, backupInfo,
+    password, targetRoomId, targetSessionId, backupInfo, opts,
 ) {
     const privKey = await keyFromAuthData(backupInfo.auth_data, password);
     return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
     );
 };
 
@@ -1722,15 +1805,16 @@ MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
  * Restores all rooms if omitted.
  * @param {string} [targetSessionId] Session ID to target a specific session.
  * Restores all sessions if omitted.
+ * @param {object} opts Optional params such as callbacks
  * @return {Promise<object>} Status of restoration with `total` and `imported`
  * key counts.
  */
 MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
-    backupInfo, targetRoomId, targetSessionId,
+    backupInfo, targetRoomId, targetSessionId, opts,
 ) {
     const privKey = decodeBase64(await this.getSecret("m.megolm_backup.v1"));
     return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
     );
 };
 
@@ -1743,20 +1827,47 @@ MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
  * @param {string} [targetSessionId] Session ID to target a specific session.
  * Restores all sessions if omitted.
  * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
+
  * @return {Promise<object>} Status of restoration with `total` and `imported`
  * key counts.
  */
 MatrixClient.prototype.restoreKeyBackupWithRecoveryKey = function(
-    recoveryKey, targetRoomId, targetSessionId, backupInfo,
+    recoveryKey, targetRoomId, targetSessionId, backupInfo, opts,
 ) {
     const privKey = decodeRecoveryKey(recoveryKey);
     return this._restoreKeyBackup(
-        privKey, targetRoomId, targetSessionId, backupInfo,
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
+    );
+};
+
+/**
+ * Restore from an existing key backup using a cached key, or fail
+ *
+ * @param {string} [targetRoomId] Room ID to target a specific room.
+ * Restores all rooms if omitted.
+ * @param {string} [targetSessionId] Session ID to target a specific session.
+ * Restores all sessions if omitted.
+ * @param {object} backupInfo Backup metadata from `checkKeyBackup`
+ * @param {object} opts Optional params such as callbacks
+ * @return {Promise<object>} Status of restoration with `total` and `imported`
+ * key counts.
+ */
+MatrixClient.prototype.restoreKeyBackupWithCache = async function(
+    targetRoomId, targetSessionId, backupInfo, opts,
+) {
+    const privKey = await this._crypto.getSessionBackupPrivateKey();
+    if (!privKey) {
+        throw new Error("Couldn't get key");
+    }
+    return this._restoreKeyBackup(
+        privKey, targetRoomId, targetSessionId, backupInfo, opts,
     );
 };
 
 MatrixClient.prototype._restoreKeyBackup = function(
     privKey, targetRoomId, targetSessionId, backupInfo,
+    { cacheCompleteCallback }={}, // For sequencing during tests
 ) {
     if (this._crypto === null) {
         throw new Error("End-to-end encryption disabled");
@@ -1784,6 +1895,13 @@ MatrixClient.prototype._restoreKeyBackup = function(
         return Promise.reject({errcode: MatrixClient.RESTORE_BACKUP_ERROR_BAD_KEY});
     }
 
+    // Cache the key, if possible.
+    // This is async.
+    this._crypto.storeSessionBackupPrivateKey(privKey)
+    .catch((e) => {
+        console.warn("Error caching session backup key:", e);
+    }).then(cacheCompleteCallback);
+
     return this._http.authedRequest(
         undefined, "GET", path.path, path.queryData, undefined,
         {prefix: PREFIX_UNSTABLE},
@@ -2381,7 +2499,7 @@ function _sendEvent(client, room, event, callback) {
         let promise;
         // this event may be queued
         if (client.scheduler) {
-            // if this returns a promsie then the scheduler has control now and will
+            // if this returns a promise then the scheduler has control now and will
             // resolve/reject when it is done. Internally, the scheduler will invoke
             // processFn which is set to this._sendEventHttpRequest so the same code
             // path is executed regardless.
@@ -2395,12 +2513,15 @@ function _sendEvent(client, room, event, callback) {
 
         if (!promise) {
             promise = _sendEventHttpRequest(client, event);
+            if (room) {
+                promise = promise.then(res => {
+                    room.updatePendingEvent(event, EventStatus.SENT, res.event_id);
+                    return res;
+                });
+            }
         }
         return promise;
     }).then(function(res) {  // the request was sent OK
-        if (room) {
-            room.updatePendingEvent(event, EventStatus.SENT, res.event_id);
-        }
         if (callback) {
             callback(null, res);
         }
@@ -4783,6 +4904,19 @@ MatrixClient.prototype.doesServerSupportSeparateAddAndBind = async function() {
         || (unstableFeatures && unstableFeatures["m.separate_add_and_bind"]);
 };
 
+/**
+ * Query the server to see if it lists support for an unstable feature
+ * in the /versions response
+ * @param {string} feature the feature name
+ * @return {Promise<boolean>} true if the feature is supported
+ */
+MatrixClient.prototype.doesServerSupportUnstableFeature = async function(feature) {
+    const response = await this.getVersions();
+    if (!response) return false;
+    const unstableFeatures = response["unstable_features"];
+    return unstableFeatures && !!unstableFeatures[feature];
+};
+
 /**
  * Get if lazy loading members is being used.
  * @return {boolean} Whether or not members are lazy loaded by this client
@@ -5138,6 +5272,15 @@ MatrixClient.prototype.getEventMapper = function() {
     return _PojoToMatrixEventMapper(this);
 };
 
+/**
+ * The app may wish to see if we have a key cached without
+ * triggering a user interaction.
+ * @return {object}
+ */
+MatrixClient.prototype.getCrossSigningCacheCallbacks = function() {
+    return this._crypto && this._crypto._crossSigningInfo.getCacheCallbacks();
+};
+
 // Identity Server Operations
 // ==========================
 
@@ -5454,13 +5597,6 @@ MatrixClient.prototype.generateClientSecret = function() {
  *     reject the key verification.
  */
 
-/**
- * Fires when a key verification started message is received.
- * @event module:client~MatrixClient#"crypto.verification.start"
- * @param {module:crypto/verification/Base} verifier a verifier object to
- *     perform the key verification
- */
-
 /**
  * Fires when a secret request has been cancelled.  If the client is prompting
  * the user to ask whether they want to share a secret, the prompt can be

src/crypto/CrossSigning.js

@@ -23,6 +23,7 @@ limitations under the License.
 import {decodeBase64, encodeBase64, pkSign, pkVerify} from './olmlib';
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
+import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
 
 function publicKeyFromKeyInfo(keyInfo) {
     // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
@@ -40,8 +41,9 @@ export class CrossSigningInfo extends EventEmitter {
      * @param {string} userId the user that the information is about
      * @param {object} callbacks Callbacks used to interact with the app
      *     Requires getCrossSigningKey and saveCrossSigningKeys
+     * @param {object} cacheCallbacks Callbacks used to interact with the cache
      */
-    constructor(userId, callbacks) {
+    constructor(userId, callbacks, cacheCallbacks) {
         super();
 
         // you can't change the userId
@@ -50,6 +52,7 @@ export class CrossSigningInfo extends EventEmitter {
             value: userId,
         });
         this._callbacks = callbacks || {};
+        this._cacheCallbacks = cacheCallbacks || {};
         this.keys = {};
         this.firstUse = true;
     }
@@ -62,6 +65,8 @@ export class CrossSigningInfo extends EventEmitter {
      * @returns {Array} An array with [ public key, Olm.PkSigning ]
      */
     async getCrossSigningKey(type, expectedPubkey) {
+        const shouldCache = ["self_signing", "user_signing"].indexOf(type) >= 0;
+
         if (!this._callbacks.getCrossSigningKey) {
             throw new Error("No getCrossSigningKey callback supplied");
         }
@@ -70,22 +75,47 @@ export class CrossSigningInfo extends EventEmitter {
             expectedPubkey = this.getId(type);
         }
 
-        const privkey = await this._callbacks.getCrossSigningKey(type, expectedPubkey);
+        function validateKey(key) {
+            if (!key) return;
+            const signing = new global.Olm.PkSigning();
+            const gotPubkey = signing.init_with_seed(key);
+            if (gotPubkey === expectedPubkey) {
+                return [gotPubkey, signing];
+            }
+            signing.free();
+        }
+
+        let privkey;
+        if (this._cacheCallbacks.getCrossSigningKeyCache && shouldCache) {
+            privkey = await this._cacheCallbacks
+              .getCrossSigningKeyCache(type, expectedPubkey);
+        }
+
+        const cacheresult = validateKey(privkey);
+        if (cacheresult) {
+            return cacheresult;
+        }
+
+        privkey = await this._callbacks.getCrossSigningKey(type, expectedPubkey);
+        const result = validateKey(privkey);
+        if (result) {
+            if (this._cacheCallbacks.storeCrossSigningKeyCache && shouldCache) {
+                await this._cacheCallbacks.storeCrossSigningKeyCache(type, privkey);
+            }
+            return result;
+        }
+
+        /* No keysource even returned a key */
         if (!privkey) {
             throw new Error(
                 "getCrossSigningKey callback for " + type + " returned falsey",
             );
         }
-        const signing = new global.Olm.PkSigning();
-        const gotPubkey = signing.init_with_seed(privkey);
-        if (gotPubkey !== expectedPubkey) {
-            signing.free();
-            throw new Error(
-                "Key type " + type + " from getCrossSigningKey callback did not match",
-            );
-        } else {
-            return [gotPubkey, signing];
-        }
+
+        /* We got some keys from the keysource, but none of them were valid */
+        throw new Error(
+            "Key type " + type + " from getCrossSigningKey callback did not match",
+        );
     }
 
     static fromStorage(obj, userId) {
@@ -111,14 +141,28 @@ export class CrossSigningInfo extends EventEmitter {
      * want to know this anyway...
      *
      * @param {SecretStorage} secretStorage The secret store using account data
-     * @returns {boolean} Whether all private keys were found in storage
+     * @returns {object} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
      */
     async isStoredInSecretStorage(secretStorage) {
-        let stored = true;
-        for (const type of ["master", "self_signing", "user_signing"]) {
-            stored &= await secretStorage.isStored(`m.cross_signing.${type}`, false);
+        // check what SSSS keys have encrypted the master key (if any)
+        const stored =
+              await secretStorage.isStored("m.cross_signing.master", false) || {};
+        // then check which of those SSSS keys have also encrypted the SSK and USK
+        function intersect(s) {
+            for (const k of Object.keys(stored)) {
+                if (!s[k]) {
+                    delete stored[k];
+                }
+            }
         }
-        return stored;
+        for (const type of ["self_signing", "user_signing"]) {
+            intersect(
+                await secretStorage.isStored(`m.cross_signing.${type}`, false) || {},
+            );
+        }
+        return Object.keys(stored).length ? stored : null;
     }
 
     /**
@@ -416,17 +460,20 @@ export class CrossSigningInfo extends EventEmitter {
      * @param {CrossSigningInfo} userCrossSigning Cross signing info for user
      * @param {module:crypto/deviceinfo} device The device to check
      * @param {bool} localTrust Whether the device is trusted locally
+     * @param {bool} trustCrossSignedDevices Whether we trust cross signed devices
      *
      * @returns {DeviceTrustLevel}
      */
-    checkDeviceTrust(userCrossSigning, device, localTrust) {
+    checkDeviceTrust(userCrossSigning, device, localTrust, trustCrossSignedDevices) {
         const userTrust = this.checkUserTrust(userCrossSigning);
 
         const userSSK = userCrossSigning.keys.self_signing;
         if (!userSSK) {
             // if the user has no self-signing key then we cannot make any
             // trust assertions about this device from cross-signing
-            return new DeviceTrustLevel(false, false, localTrust);
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
         }
 
         const deviceObj = deviceToObject(device, userCrossSigning.userId);
@@ -438,11 +485,22 @@ export class CrossSigningInfo extends EventEmitter {
                 deviceObj, publicKeyFromKeyInfo(userSSK), userCrossSigning.userId,
             );
             // ...then we trust this device as much as far as we trust the user
-            return DeviceTrustLevel.fromUserTrustLevel(userTrust, localTrust);
+            return DeviceTrustLevel.fromUserTrustLevel(
+                userTrust, localTrust, trustCrossSignedDevices,
+            );
         } catch (e) {
-            return new DeviceTrustLevel(false, false, localTrust);
+            return new DeviceTrustLevel(
+                false, false, localTrust, trustCrossSignedDevices,
+            );
         }
     }
+
+    /**
+     * @returns {object} Cache callbacks
+     */
+    getCacheCallbacks() {
+        return this._cacheCallbacks;
+    }
 }
 
 function deviceToObject(device, userId) {
@@ -496,17 +554,19 @@ export class UserTrustLevel {
  * Represents the ways in which we trust a device
  */
 export class DeviceTrustLevel {
-    constructor(crossSigningVerified, tofu, localVerified) {
+    constructor(crossSigningVerified, tofu, localVerified, trustCrossSignedDevices) {
         this._crossSigningVerified = crossSigningVerified;
         this._tofu = tofu;
         this._localVerified = localVerified;
+        this._trustCrossSignedDevices = trustCrossSignedDevices;
     }
 
-    static fromUserTrustLevel(userTrustLevel, localVerified) {
+    static fromUserTrustLevel(userTrustLevel, localVerified, trustCrossSignedDevices) {
         return new DeviceTrustLevel(
             userTrustLevel._crossSigningVerified,
             userTrustLevel._tofu,
             localVerified,
+            trustCrossSignedDevices,
         );
     }
 
@@ -514,7 +574,9 @@ export class DeviceTrustLevel {
      * @returns {bool} true if this device is verified via any means
      */
     isVerified() {
-        return this.isCrossSigningVerified() || this.isLocallyVerified();
+        return Boolean(this.isLocallyVerified() || (
+            this._trustCrossSignedDevices && this.isCrossSigningVerified()
+        ));
     }
 
     /**
@@ -539,3 +601,28 @@ export class DeviceTrustLevel {
         return this._tofu;
     }
 }
+
+export function createCryptoStoreCacheCallbacks(store) {
+    return {
+        getCrossSigningKeyCache: function(type, _expectedPublicKey) {
+            return new Promise((resolve) => {
+                return store.doTxn(
+                    'readonly',
+                    [IndexedDBCryptoStore.STORE_ACCOUNT],
+                    (txn) => {
+                        store.getSecretStorePrivateKey(txn, resolve, type);
+                    },
+                );
+            });
+        },
+        storeCrossSigningKeyCache: function(type, key) {
+            return store.doTxn(
+                'readwrite',
+                [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    store.storeSecretStorePrivateKey(txn, type, key);
+                },
+            );
+        },
+    };
+}

src/crypto/DeviceList.js

@@ -197,16 +197,16 @@ export class DeviceList extends EventEmitter {
             const resolveSavePromise = this._resolveSavePromise;
             this._savePromiseTime = targetTime;
             this._saveTimer = setTimeout(() => {
-                logger.log('Saving device tracking data at token ' + this._syncToken);
+                logger.log('Saving device tracking data', this._syncToken);
+
                 // null out savePromise now (after the delay but before the write),
                 // otherwise we could return the existing promise when the save has
-                // actually already happened. Likewise for the dirty flag.
+                // actually already happened.
                 this._savePromiseTime = null;
                 this._saveTimer = null;
                 this._savePromise = null;
                 this._resolveSavePromise = null;
 
-                this._dirty = false;
                 this._cryptoStore.doTxn(
                     'readwrite', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                         this._cryptoStore.storeEndToEndDeviceData({
@@ -217,7 +217,13 @@ export class DeviceList extends EventEmitter {
                         }, txn);
                     },
                 ).then(() => {
+                    // The device list is considered dirty until the write
+                    // completes.
+                    this._dirty = false;
                     resolveSavePromise();
+                }, err => {
+                    logger.error('Failed to save device tracking data', this._syncToken);
+                    logger.error(err);
                 });
             }, delay);
         }
@@ -311,6 +317,15 @@ export class DeviceList extends EventEmitter {
         return stored;
     }
 
+    /**
+     * Returns a list of all user IDs the DeviceList knows about
+     *
+     * @return {array} All known user IDs
+     */
+    getKnownUserIds() {
+        return Object.keys(this._devices);
+    }
+
     /**
      * Get the stored device keys for a user id
      *
@@ -373,6 +388,26 @@ export class DeviceList extends EventEmitter {
         return DeviceInfo.fromStorage(devs[deviceId], deviceId);
     }
 
+    /**
+     * Get a user ID by one of their device's curve25519 identity key
+     *
+     * @param {string} algorithm  encryption algorithm
+     * @param {string} senderKey  curve25519 key to match
+     *
+     * @return {string} user ID
+     */
+    getUserByIdentityKey(algorithm, senderKey) {
+        if (
+            algorithm !== olmlib.OLM_ALGORITHM &&
+            algorithm !== olmlib.MEGOLM_ALGORITHM
+        ) {
+            // we only deal in olm keys
+            return null;
+        }
+
+        return this._userByIdentityKey[senderKey];
+    }
+
     /**
      * Find a device by curve25519 identity key
      *
@@ -382,19 +417,11 @@ export class DeviceList extends EventEmitter {
      * @return {module:crypto/deviceinfo?}
      */
     getDeviceByIdentityKey(algorithm, senderKey) {
-        const userId = this._userByIdentityKey[senderKey];
+        const userId = this.getUserByIdentityKey(algorithm, senderKey);
         if (!userId) {
             return null;
         }
 
-        if (
-            algorithm !== olmlib.OLM_ALGORITHM &&
-            algorithm !== olmlib.MEGOLM_ALGORITHM
-        ) {
-            // we only deal in olm keys
-            return null;
-        }
-
         const devices = this._devices[userId];
         if (!devices) {
             return null;
@@ -749,31 +776,34 @@ class DeviceListUpdateSerialiser {
 
         this._baseApis.downloadKeysForUsers(
             downloadUsers, opts,
-        ).then((res) => {
+        ).then(async (res) => {
             const dk = res.device_keys || {};
             const masterKeys = res.master_keys || {};
             const ssks = res.self_signing_keys || {};
             const usks = res.user_signing_keys || {};
 
-            // do each user in a separate promise, to avoid wedging the CPU
+            // yield to other things that want to execute in between users, to
+            // avoid wedging the CPU
             // (https://github.com/vector-im/riot-web/issues/3158)
             //
             // of course we ought to do this in a web worker or similar, but
             // this serves as an easy solution for now.
-            let prom = Promise.resolve();
             for (const userId of downloadUsers) {
-                prom = prom.then(sleep(5)).then(() => {
-                    return this._processQueryResponseForUser(
+                await sleep(5);
+                try {
+                    await this._processQueryResponseForUser(
                         userId, dk[userId], {
                             master: masterKeys[userId],
                             self_signing: ssks[userId],
                             user_signing: usks[userId],
                         },
                     );
-                });
+                } catch (e) {
+                    // log the error but continue, so that one bad key
+                    // doesn't kill the whole process
+                    logger.error(`Error processing keys for ${userId}:`, e);
+                }
             }
-
-            return prom;
         }).then(() => {
             logger.log('Completed key download for ' + downloadUsers);
 

src/crypto/SecretStorage.js

@@ -1,5 +1,5 @@
 /*
-Copyright 2019 The Matrix.org Foundation C.I.C.
+Copyright 2019, 2020 The Matrix.org Foundation C.I.C.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -19,8 +19,13 @@ import {logger} from '../logger';
 import * as olmlib from './olmlib';
 import {pkVerify} from './olmlib';
 import {randomString} from '../randomstring';
+import {encryptAES, decryptAES} from './aes';
 
-export const SECRET_STORAGE_ALGORITHM_V1 = "m.secret_storage.v1.curve25519-aes-sha2";
+export const SECRET_STORAGE_ALGORITHM_V1_AES
+    = "m.secret_storage.v1.aes-hmac-sha2";
+// don't use curve25519 for writing data.
+export const SECRET_STORAGE_ALGORITHM_V1_CURVE25519
+    = "m.secret_storage.v1.curve25519-aes-sha2";
 
 /**
  * Implements Secure Secret Storage and Sharing (MSC1946)
@@ -85,20 +90,12 @@ export class SecretStorage extends EventEmitter {
         }
 
         switch (algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1:
+        case SECRET_STORAGE_ALGORITHM_V1_AES:
         {
             const decryption = new global.Olm.PkDecryption();
             try {
-                const { passphrase, pubkey } = opts;
-                // Copies in public key details of the form generated by
-                // the Crypto module's `createRecoveryKeyFromPassphrase`.
-                if (passphrase && pubkey) {
-                    keyData.passphrase = passphrase;
-                    keyData.pubkey = pubkey;
-                } else if (pubkey) {
-                    keyData.pubkey = pubkey;
-                } else {
-                    keyData.pubkey = decryption.generate_key();
+                if (opts.passphrase) {
+                    keyData.passphrase = opts.passphrase;
                 }
             } finally {
                 decryption.free();
@@ -155,6 +152,28 @@ export class SecretStorage extends EventEmitter {
         );
     }
 
+    /**
+     * Get the key information for a given ID.
+     *
+     * @param {string} [keyId = default key's ID] The ID of the key to check
+     *     for. Defaults to the default key ID if not provided.
+     * @returns {Array?} If the key was found, the return value is an array of
+     *     the form [keyId, keyInfo].  Otherwise, null is returned.
+     */
+    async getKey(keyId) {
+        if (!keyId) {
+            keyId = await this.getDefaultKeyId();
+        }
+        if (!keyId) {
+            return null;
+        }
+
+        const keyInfo = await this._baseApis.getAccountDataFromServer(
+            "m.secret_storage.key." + keyId,
+        );
+        return keyInfo ? [keyId, keyInfo] : null;
+    }
+
     /**
      * Check whether we have a key with a given ID.
      *
@@ -163,16 +182,16 @@ export class SecretStorage extends EventEmitter {
      * @return {boolean} Whether we have the key.
      */
     async hasKey(keyId) {
-        if (!keyId) {
-            keyId = await this.getDefaultKeyId();
-        }
-        if (!keyId) {
+        return !!(await this.getKey(keyId));
+    }
+
+    async keyNeedsUpgrade(keyId) {
+        const keyInfo = await this.getKey(keyId);
+        if (keyInfo && keyInfo[1].algorithm === SECRET_STORAGE_ALGORITHM_V1_CURVE25519) {
+            return true;
+        } else {
             return false;
         }
-
-        return !!this._baseApis.getAccountDataFromServer(
-            "m.secret_storage.key." + keyId,
-        );
     }
 
     /**
@@ -207,24 +226,13 @@ export class SecretStorage extends EventEmitter {
                 throw new Error("Unknown key: " + keyId);
             }
 
-            // check signature of key info
-            pkVerify(
-                keyInfo,
-                this._crossSigningInfo.getId('master'),
-                this._crossSigningInfo.userId,
-            );
-
             // encrypt secret, based on the algorithm
             switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
+            case SECRET_STORAGE_ALGORITHM_V1_AES:
             {
-                const encryption = new global.Olm.PkEncryption();
-                try {
-                    encryption.set_recipient_key(keyInfo.pubkey);
-                    encrypted[keyId] = encryption.encrypt(secret);
-                } finally {
-                    encryption.free();
-                }
+                const keys = {[keyId]: keyInfo};
+                const [, encryption] = await this._getSecretStorageKey(keys, name);
+                encrypted[keyId] = await encryption.encrypt(secret);
                 break;
             }
             default:
@@ -238,29 +246,6 @@ export class SecretStorage extends EventEmitter {
         await this._baseApis.setAccountData(name, {encrypted});
     }
 
-    /**
-     * Store a secret defined to be the same as the given key.
-     * No secret information will be stored, instead the secret will
-     * be stored with a marker to say that the contents of the secret is
-     * the value of the given key.
-     * This is useful for migration from systems that predate SSSS such as
-     * key backup.
-     *
-     * @param {string} name The name of the secret
-     * @param {string} keyId The ID of the key whose value will be the
-     *     value of the secret
-     * @returns {Promise} resolved when account data is saved
-     */
-    storePassthrough(name, keyId) {
-        return this._baseApis.setAccountData(name, {
-            encrypted: {
-                [keyId]: {
-                    passthrough: true,
-                },
-            },
-        });
-    }
-
     /**
      * Temporary method to fix up existing accounts where secrets
      * are incorrectly stored without the 'encrypted' level
@@ -317,7 +302,12 @@ export class SecretStorage extends EventEmitter {
             );
             const encInfo = secretInfo.encrypted[keyId];
             switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
+            case SECRET_STORAGE_ALGORITHM_V1_AES:
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    keys[keyId] = keyInfo;
+                }
+                break;
+            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
                 if (
                     keyInfo.pubkey && (
                         (encInfo.ciphertext && encInfo.mac && encInfo.ephemeral) ||
@@ -344,15 +334,9 @@ export class SecretStorage extends EventEmitter {
             // since we just want to return the key itself.
             if (encInfo.passthrough) return decryption.get_private_key();
 
-            // decrypt secret
-            switch (keys[keyId].algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-                return decryption.decrypt(
-                    encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
-                );
-            }
+            return await decryption.decrypt(encInfo);
         } finally {
-            if (decryption) decryption.free();
+            if (decryption && decryption.free) decryption.free();
         }
     }
 
@@ -362,22 +346,26 @@ export class SecretStorage extends EventEmitter {
      * @param {string} name the name of the secret
      * @param {boolean} checkKey check if the secret is encrypted by a trusted key
      *
-     * @return {boolean} whether or not the secret is stored
+     * @return {object?} map of key name to key info the secret is encrypted
+     *     with, or null if it is not present or not encrypted with a trusted
+     *     key
      */
     async isStored(name, checkKey) {
         // check if secret exists
         let secretInfo = await this._baseApis.getAccountDataFromServer(name);
-        if (!secretInfo) return false;
+        if (!secretInfo) return null;
         if (!secretInfo.encrypted) {
             // try to fix it up
             secretInfo = await this._fixupStoredSecret(name, secretInfo);
             if (!secretInfo || !secretInfo.encrypted) {
-                return false;
+                return null;
             }
         }
 
         if (checkKey === undefined) checkKey = true;
 
+        const ret = {};
+
         // check if secret is encrypted by a known/trusted secret and
         // encryption looks sane
         for (const keyId of Object.keys(secretInfo.encrypted)) {
@@ -385,32 +373,55 @@ export class SecretStorage extends EventEmitter {
             const keyInfo = await this._baseApis.getAccountDataFromServer(
                 "m.secret_storage.key." + keyId,
             );
-            if (!keyInfo) return false;
+            if (!keyInfo) continue;
             const encInfo = secretInfo.encrypted[keyId];
-            if (checkKey) {
-                pkVerify(
-                    keyInfo,
-                    this._crossSigningInfo.getId('master'),
-                    this._crossSigningInfo.userId,
-                );
-            }
 
             // We don't actually need the decryption object if it's a passthrough
             // since we just want to return the key itself.
-            if (encInfo.passthrough) return true;
+            if (encInfo.passthrough) {
+                try {
+                    pkVerify(
+                        keyInfo,
+                        this._crossSigningInfo.getId('master'),
+                        this._crossSigningInfo.userId,
+                    );
+                } catch (e) {
+                    // not trusted, so move on to the next key
+                    continue;
+                }
+                ret[keyId] = keyInfo;
+                continue;
+            }
 
             switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
+            case SECRET_STORAGE_ALGORITHM_V1_AES:
+                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
+                    ret[keyId] = keyInfo;
+                }
+                break;
+            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
                 if (keyInfo.pubkey && encInfo.ciphertext && encInfo.mac
                     && encInfo.ephemeral) {
-                    return true;
+                    if (checkKey) {
+                        try {
+                            pkVerify(
+                                keyInfo,
+                                this._crossSigningInfo.getId('master'),
+                                this._crossSigningInfo.userId,
+                            );
+                        } catch (e) {
+                            // not trusted, so move on to the next key
+                            continue;
+                        }
+                    }
+                    ret[keyId] = keyInfo;
                 }
                 break;
             default:
                 // do nothing if we don't understand the encryption algorithm
             }
         }
-        return false;
+        return Object.keys(ret).length ? ret : null;
     }
 
     /**
@@ -462,6 +473,7 @@ export class SecretStorage extends EventEmitter {
         for (const device of devices) {
             toDevice[device] = requestData;
         }
+        logger.info(`Request secret ${name} from ${devices}, id ${requestId}`);
         this._baseApis.sendToDevice("m.secret.request", {
             [this._baseApis.getUserId()]: toDevice,
         });
@@ -515,6 +527,7 @@ export class SecretStorage extends EventEmitter {
                 device_trust: this._baseApis.checkDeviceTrust(sender, deviceId),
             });
             if (secret) {
+                logger.info(`Preparing ${content.name} secret for ${deviceId}`);
                 const payload = {
                     type: "m.secret.send",
                     content: {
@@ -551,7 +564,10 @@ export class SecretStorage extends EventEmitter {
                     },
                 };
 
+                logger.info(`Sending ${content.name} secret for ${deviceId}`);
                 this._baseApis.sendToDevice("m.room.encrypted", contentMap);
+            } else {
+                logger.info(`Request denied for ${content.name} secret for ${deviceId}`);
             }
         }
     }
@@ -563,7 +579,7 @@ export class SecretStorage extends EventEmitter {
             return;
         }
         const content = event.getContent();
-        logger.log("got secret share for request ", content.request_id);
+        logger.log("got secret share for request", content.request_id);
         const requestControl = this._requests[content.request_id];
         if (requestControl) {
             // make sure that the device that sent it is one of the devices that
@@ -607,26 +623,48 @@ export class SecretStorage extends EventEmitter {
         }
 
         switch (keys[keyId].algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1:
-            {
-                const decryption = new global.Olm.PkDecryption();
-                let pubkey;
-                try {
-                    pubkey = decryption.init_with_private_key(privateKey);
-                } catch (e) {
-                    decryption.free();
-                    throw new Error("getSecretStorageKey callback returned invalid key");
-                }
-                if (pubkey !== keys[keyId].pubkey) {
-                    decryption.free();
-                    throw new Error(
-                        "getSecretStorageKey callback returned incorrect key",
-                    );
-                }
-                return [keyId, decryption];
+        case SECRET_STORAGE_ALGORITHM_V1_AES:
+        {
+            const decryption = {
+                encrypt: async function(secret) {
+                    return await encryptAES(secret, privateKey, name);
+                },
+                decrypt: async function(encInfo) {
+                    return await decryptAES(encInfo, privateKey, name);
+                },
+            };
+            return [keyId, decryption];
+        }
+        case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
+        {
+            const pkDecryption = new global.Olm.PkDecryption();
+            let pubkey;
+            try {
+                pubkey = pkDecryption.init_with_private_key(privateKey);
+            } catch (e) {
+                pkDecryption.free();
+                throw new Error("getSecretStorageKey callback returned invalid key");
             }
-            default:
-                throw new Error("Unknown key type: " + keys[keyId].algorithm);
+            if (pubkey !== keys[keyId].pubkey) {
+                pkDecryption.free();
+                throw new Error(
+                    "getSecretStorageKey callback returned incorrect key",
+                );
+            }
+            const decryption = {
+                free: pkDecryption.free.bind(pkDecryption),
+                decrypt: async function(encInfo) {
+                    return pkDecryption.decrypt(
+                        encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
+                    );
+                },
+                // needed for passthrough
+                get_private_key: pkDecryption.get_private_key.bind(pkDecryption),
+            };
+            return [keyId, decryption];
+        }
+        default:
+            throw new Error("Unknown key type: " + keys[keyId].algorithm);
         }
     }
 }

src/crypto/aes.js

@@ -0,0 +1,239 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {getCrypto} from '../utils';
+import {decodeBase64, encodeBase64} from './olmlib';
+
+const subtleCrypto = (typeof window !== "undefined" && window.crypto) ?
+    (window.crypto.subtle || window.crypto.webkitSubtle) : null;
+
+// salt for HKDF, with 8 bytes of zeros
+const zerosalt = new Uint8Array(8);
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function encryptNode(data, key, name) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    const iv = crypto.randomBytes(16);
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const cipher = crypto.createCipheriv("aes-256-ctr", aesKey, iv);
+    const ciphertext = cipher.update(data, "utf-8", "base64")
+          + cipher.final("base64");
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(ciphertext, "base64").digest("base64");
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: ciphertext,
+        mac: hmac,
+    };
+}
+
+/**
+ * decrypt a string in Node.js
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptNode(data, key, name) {
+    const crypto = getCrypto();
+    if (!crypto) {
+        throw new Error("No usable crypto implementation");
+    }
+
+    const [aesKey, hmacKey] = deriveKeysNode(key, name);
+
+    const hmac = crypto.createHmac("sha256", hmacKey)
+        .update(data.ciphertext, "base64").digest("base64");
+
+    if (hmac !== data.mac) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const decipher = crypto.createDecipheriv(
+        "aes-256-ctr", aesKey, decodeBase64(data.iv),
+    );
+    return decipher.update(data.ciphertext, "base64", "utf-8")
+          + decipher.final("utf-8");
+}
+
+function deriveKeysNode(key, name) {
+    const crypto = getCrypto();
+    const prk = crypto.createHmac("sha256", zerosalt)
+        .update(key).digest();
+
+    const b = Buffer.alloc(1, 1);
+    const aesKey = crypto.createHmac("sha256", prk)
+        .update(name, "utf-8").update(b).digest();
+    b[0] = 2;
+    const hmacKey = crypto.createHmac("sha256", prk)
+        .update(aesKey).update(name, "utf-8").update(b).digest();
+
+    return [aesKey, hmacKey];
+}
+
+/**
+ * encrypt a string in Node.js
+ *
+ * @param {string} data the plaintext to encrypt
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function encryptBrowser(data, key, name) {
+    const iv = new Uint8Array(16);
+    window.crypto.getRandomValues(iv);
+
+    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
+    // (which would mean we wouldn't be able to decrypt on Android). The loss
+    // of a single bit of iv is a price we have to pay.
+    iv[8] &= 0x7f;
+
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+    const encodedData = new TextEncoder().encode(data);
+
+    const ciphertext = await subtleCrypto.encrypt(
+        {
+            name: "AES-CTR",
+            counter: iv,
+            length: 64,
+        },
+        aesKey,
+        encodedData,
+    );
+
+    const hmac = await subtleCrypto.sign(
+        {name: 'HMAC'},
+        hmacKey,
+        ciphertext,
+    );
+
+    return {
+        iv: encodeBase64(iv),
+        ciphertext: encodeBase64(ciphertext),
+        mac: encodeBase64(hmac),
+    };
+}
+
+/**
+ * decrypt a string in the browser
+ *
+ * @param {object} data the encrypted data
+ * @param {string} data.ciphertext the ciphertext in base64
+ * @param {string} data.iv the initialization vector in base64
+ * @param {string} data.mac the HMAC in base64
+ * @param {Uint8Array} key the encryption key to use
+ * @param {string} name the name of the secret
+ */
+async function decryptBrowser(data, key, name) {
+    const [aesKey, hmacKey] = await deriveKeysBrowser(key, name);
+
+    const ciphertext = decodeBase64(data.ciphertext);
+
+    if (!await subtleCrypto.verify(
+        {name: "HMAC"},
+        hmacKey,
+        decodeBase64(data.mac),
+        ciphertext,
+    )) {
+        throw new Error(`Error decrypting secret ${name}: bad MAC`);
+    }
+
+    const plaintext = await subtleCrypto.decrypt(
+        {
+            name: "AES-CTR",
+            counter: decodeBase64(data.iv),
+            length: 64,
+        },
+        aesKey,
+        ciphertext,
+    );
+
+    return new TextDecoder().decode(new Uint8Array(plaintext));
+}
+
+async function deriveKeysBrowser(key, name) {
+    const hkdfkey = await subtleCrypto.importKey(
+        'raw',
+        key,
+        {name: "HKDF"},
+        false,
+        ["deriveBits"],
+    );
+    const keybits = await subtleCrypto.deriveBits(
+        {
+            name: "HKDF",
+            salt: zerosalt,
+            info: (new TextEncoder().encode(name)),
+            hash: "SHA-256",
+        },
+        hkdfkey,
+        512,
+    );
+
+    const aesKey = keybits.slice(0, 32);
+    const hmacKey = keybits.slice(32);
+
+    const aesProm = subtleCrypto.importKey(
+        'raw',
+        aesKey,
+        {name: 'AES-CTR'},
+        false,
+        ['encrypt', 'decrypt'],
+    );
+
+    const hmacProm = subtleCrypto.importKey(
+        'raw',
+        hmacKey,
+        {
+            name: 'HMAC',
+            hash: {name: 'SHA-256'},
+        },
+        false,
+        ['sign', 'verify'],
+    );
+
+    return await Promise.all([aesProm, hmacProm]);
+}
+
+export function encryptAES(...args) {
+    return subtleCrypto ? encryptBrowser(...args) : encryptNode(...args);
+}
+
+export function decryptAES(...args) {
+    return subtleCrypto ? decryptBrowser(...args) : decryptNode(...args);
+}
+

src/crypto/algorithms/base.js

@@ -60,6 +60,15 @@ export class EncryptionAlgorithm {
         this._roomId = params.roomId;
     }
 
+    /**
+     * Perform any background tasks that can be done before a message is ready to
+     * send, in order to speed up sending of the message.
+     *
+     * @param {module:models/room} room the room the event is in
+     */
+    prepareToEncrypt(room) {
+    }
+
     /**
      * Encrypt a message event
      *

src/crypto/algorithms/megolm.js

@@ -185,15 +185,15 @@ utils.inherits(MegolmEncryption, EncryptionAlgorithm);
  *
  * @param {Object} devicesInRoom The devices in this room, indexed by user ID
  * @param {Object} blocked The devices that are blocked, indexed by user ID
+ * @param {boolean} [singleOlmCreationPhase] Only perform one round of olm
+ *     session creation
  *
  * @return {Promise} Promise which resolves to the
  *    OutboundSessionInfo when setup is complete.
  */
 MegolmEncryption.prototype._ensureOutboundSession = async function(
-    devicesInRoom, blocked,
+    devicesInRoom, blocked, singleOlmCreationPhase,
 ) {
-    const self = this;
-
     let session;
 
     // takes the previous OutboundSessionInfo, and considers whether to create
@@ -201,12 +201,12 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
     // Updates `session` to hold the final OutboundSessionInfo.
     //
     // returns a promise which resolves once the keyshare is successful.
-    async function prepareSession(oldSession) {
+    const prepareSession = async (oldSession) => {
         session = oldSession;
 
         // need to make a brand new session?
-        if (session && session.needsRotation(self._sessionRotationPeriodMsgs,
-                                             self._sessionRotationPeriodMs)
+        if (session && session.needsRotation(this._sessionRotationPeriodMsgs,
+                                             this._sessionRotationPeriodMs)
            ) {
             logger.log("Starting new megolm session because we need to rotate.");
             session = null;
@@ -218,32 +218,20 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
         }
 
         if (!session) {
-            logger.log(`Starting new megolm session for room ${self._roomId}`);
-            session = await self._prepareNewSession();
+            logger.log(`Starting new megolm session for room ${this._roomId}`);
+            session = await this._prepareNewSession();
             logger.log(`Started new megolm session ${session.sessionId} ` +
-                       `for room ${self._roomId}`);
-            self._outboundSessions[session.sessionId] = session;
+                       `for room ${this._roomId}`);
+            this._outboundSessions[session.sessionId] = session;
         }
 
         // now check if we need to share with any devices
         const shareMap = {};
 
-        for (const userId in devicesInRoom) {
-            if (!devicesInRoom.hasOwnProperty(userId)) {
-                continue;
-            }
-
-            const userDevices = devicesInRoom[userId];
-
-            for (const deviceId in userDevices) {
-                if (!userDevices.hasOwnProperty(deviceId)) {
-                    continue;
-                }
-
-                const deviceInfo = userDevices[deviceId];
-
+        for (const [userId, userDevices] of Object.entries(devicesInRoom)) {
+            for (const [deviceId, deviceInfo] of Object.entries(userDevices)) {
                 const key = deviceInfo.getIdentityKey();
-                if (key == self._olmDevice.deviceCurve25519Key) {
+                if (key == this._olmDevice.deviceCurve25519Key) {
                     // don't bother sending to ourself
                     continue;
                 }
@@ -258,51 +246,99 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
             }
         }
 
-        const errorDevices = [];
-
-        await self._shareKeyWithDevices(
-            session, shareMap, errorDevices,
+        const key = this._olmDevice.getOutboundGroupSessionKey(session.sessionId);
+        const payload = {
+            type: "m.room_key",
+            content: {
+                algorithm: olmlib.MEGOLM_ALGORITHM,
+                room_id: this._roomId,
+                session_id: session.sessionId,
+                session_key: key.key,
+                chain_index: key.chain_index,
+            },
+        };
+        const [devicesWithoutSession, olmSessions] = await olmlib.getExistingOlmSessions(
+            this._olmDevice, this._baseApis, shareMap,
         );
 
-        // are there any new blocked devices that we need to notify?
-        const blockedMap = {};
-        for (const userId in blocked) {
-            if (!blocked.hasOwnProperty(userId)) {
-                continue;
-            }
+        await Promise.all([
+            (async () => {
+                // share keys with devices that we already have a session for
+                await this._shareKeyWithOlmSessions(
+                    session, key, payload, olmSessions,
+                );
+            })(),
+            (async () => {
+                const errorDevices = [];
 
-            const userBlockedDevices = blocked[userId];
+                // meanwhile, establish olm sessions for devices that we don't
+                // already have a session for, and share keys with them.  If
+                // we're doing two phases of olm session creation, use a
+                // shorter timeout when fetching one-time keys for the first
+                // phase.
+                const start = Date.now();
+                const failedServers = [];
+                await this._shareKeyWithDevices(
+                    session, key, payload, devicesWithoutSession, errorDevices,
+                    singleOlmCreationPhase ? 10000 : 2000, failedServers,
+                );
 
-            for (const deviceId in userBlockedDevices) {
-                if (!userBlockedDevices.hasOwnProperty(deviceId)) {
-                    continue;
+                if (!singleOlmCreationPhase && (Date.now() - start < 10000)) {
+                    // perform the second phase of olm session creation if requested,
+                    // and if the first phase didn't take too long
+                    (async () => {
+                        // Retry sending keys to devices that we were unable to establish
+                        // an olm session for.  This time, we use a longer timeout, but we
+                        // do this in the background and don't block anything else while we
+                        // do this.  We only need to retry users from servers that didn't
+                        // respond the first time.
+                        const retryDevices = {};
+                        const failedServerMap = new Set;
+                        for (const server of failedServers) {
+                            failedServerMap.add(server);
+                        }
+                        const failedDevices = [];
+                        for (const {userId, deviceInfo} of errorDevices) {
+                            const userHS = userId.slice(userId.indexOf(":") + 1);
+                            if (failedServerMap.has(userHS)) {
+                                retryDevices[userId] = retryDevices[userId] || [];
+                                retryDevices[userId].push(deviceInfo);
+                            } else {
+                                // if we aren't going to retry, then handle it
+                                // as a failed device
+                                failedDevices.push({userId, deviceInfo});
+                            }
+                        }
+
+                        await this._shareKeyWithDevices(
+                            session, key, payload, retryDevices, failedDevices,
+                        );
+
+                        await this._notifyFailedOlmDevices(session, key, failedDevices);
+                    })();
+                } else {
+                    await this._notifyFailedOlmDevices(session, key, errorDevices);
+                }
+            })(),
+            (async () => {
+                // also, notify blocked devices that they're blocked
+                const blockedMap = {};
+                for (const [userId, userBlockedDevices] of Object.entries(blocked)) {
+                    for (const [deviceId, device] of Object.entries(userBlockedDevices)) {
+                        if (
+                            !session.blockedDevicesNotified[userId] ||
+                            session.blockedDevicesNotified[userId][deviceId] === undefined
+                        ) {
+                            blockedMap[userId] = blockedMap[userId] || {};
+                            blockedMap[userId][deviceId] = {device};
+                        }
+                    }
                 }
 
-                if (
-                    !session.blockedDevicesNotified[userId] ||
-                        session.blockedDevicesNotified[userId][deviceId] === undefined
-                ) {
-                    blockedMap[userId] = blockedMap[userId] || [];
-                    blockedMap[userId].push(userBlockedDevices[deviceId]);
-                }
-            }
-        }
-
-
-        const filteredErrorDevices =
-              await self._olmDevice.filterOutNotifiedErrorDevices(errorDevices);
-        for (const {userId, deviceInfo} of filteredErrorDevices) {
-            blockedMap[userId] = blockedMap[userId] || [];
-            blockedMap[userId].push({
-                code: "m.no_olm",
-                reason: WITHHELD_MESSAGES["m.no_olm"],
-                deviceInfo,
-            });
-        }
-
-        // notify blocked devices that they're blocked
-        await self._notifyBlockedDevices(session, blockedMap);
-    }
+                await this._notifyBlockedDevices(session, blockedMap);
+            })(),
+        ]);
+    };
 
     // helper which returns the session prepared by prepareSession
     function returnSession() {
@@ -349,96 +385,47 @@ MegolmEncryption.prototype._prepareNewSession = async function() {
 };
 
 /**
- * Splits the user device map into multiple chunks to reduce the number of
- * devices we encrypt to per API call. Also filters out devices we don't have
- * a session with.
+ * Determines what devices in devicesByUser don't have an olm session as given
+ * in devicemap.
  *
  * @private
  *
- * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ * @param {object} devicemap the devices that have olm sessions, as returned by
+ *     olmlib.ensureOlmSessionsForDevices.
+ * @param {object} devicesByUser a map of user IDs to array of deviceInfo
+ * @param {array} [noOlmDevices] an array to fill with devices that don't have
+ *     olm sessions
  *
- * @param {number} chainIndex current chain index
- *
- * @param {object<userId, deviceId>} devicemap
- *   mapping from userId to deviceId to {@link module:crypto~OlmSessionResult}
- *
- * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
- *    map from userid to list of devices
- *
- * @param {array<object>} errorDevices
- *    array that will be populated with the devices that can't get an
- *    olm session for
- *
- * @return {array<object<userid, deviceInfo>>}
+ * @return {array} an array of devices that don't have olm sessions.  If
+ *     noOlmDevices is specified, then noOlmDevices will be returned.
  */
-MegolmEncryption.prototype._splitUserDeviceMap = function(
-    session, chainIndex, devicemap, devicesByUser, errorDevices,
+MegolmEncryption.prototype._getDevicesWithoutSessions = function(
+    devicemap, devicesByUser, noOlmDevices,
 ) {
-    const maxUsersPerRequest = 20;
+    noOlmDevices = noOlmDevices || [];
 
-    // use an array where the slices of a content map gets stored
-    const mapSlices = [];
-    let currentSliceId = 0; // start inserting in the first slice
-    let entriesInCurrentSlice = 0;
-
-    for (const userId of Object.keys(devicesByUser)) {
-        const devicesToShareWith = devicesByUser[userId];
+    for (const [userId, devicesToShareWith] of Object.entries(devicesByUser)) {
         const sessionResults = devicemap[userId];
 
-        for (let i = 0; i < devicesToShareWith.length; i++) {
-            const deviceInfo = devicesToShareWith[i];
+        for (const deviceInfo of devicesToShareWith) {
             const deviceId = deviceInfo.deviceId;
 
             const sessionResult = sessionResults[deviceId];
             if (!sessionResult.sessionId) {
                 // no session with this device, probably because there
                 // were no one-time keys.
-                //
-                // we could send them a to_device message anyway, as a
-                // signal that they have missed out on the key sharing
-                // message because of the lack of keys, but there's not
-                // much point in that really; it will mostly serve to clog
-                // up to_device inboxes.
 
-                // mark this device as "handled" because we don't want to try
-                // to claim a one-time-key for dead devices on every message.
-                session.markSharedWithDevice(userId, deviceId, chainIndex);
-
-                errorDevices.push({userId, deviceInfo});
+                noOlmDevices.push({userId, deviceInfo});
+                delete sessionResults[deviceId];
 
                 // ensureOlmSessionsForUsers has already done the logging,
                 // so just skip it.
                 continue;
             }
-
-            logger.log(
-                "share keys with device " + userId + ":" + deviceId,
-            );
-
-            if (!mapSlices[currentSliceId]) {
-                mapSlices[currentSliceId] = [];
-            }
-
-            mapSlices[currentSliceId].push({
-                userId: userId,
-                deviceInfo: deviceInfo,
-            });
-
-            entriesInCurrentSlice++;
-        }
-
-        // We do this in the per-user loop as we prefer that all messages to the
-        // same user end up in the same API call to make it easier for the
-        // server (e.g. only have to send one EDU if a remote user, etc). This
-        // does mean that if a user has many devices we may go over the desired
-        // limit, but its not a hard limit so that is fine.
-        if (entriesInCurrentSlice > maxUsersPerRequest) {
-            // the current slice is filled up. Start inserting into the next slice
-            entriesInCurrentSlice = 0;
-            currentSliceId++;
         }
     }
-    return mapSlices;
+
+    return noOlmDevices;
 };
 
 /**
@@ -451,20 +438,18 @@ MegolmEncryption.prototype._splitUserDeviceMap = function(
  *
  * @return {array<array<object>>} the blocked devices, split into chunks
  */
-MegolmEncryption.prototype._splitBlockedDevices = function(devicesByUser) {
-    const maxUsersPerRequest = 20;
+MegolmEncryption.prototype._splitDevices = function(devicesByUser) {
+    const maxDevicesPerRequest = 20;
 
     // use an array where the slices of a content map gets stored
     let currentSlice = [];
     const mapSlices = [currentSlice];
 
-    for (const userId of Object.keys(devicesByUser)) {
-        const userBlockedDevicesToShareWith = devicesByUser[userId];
-
-        for (const blockedInfo of userBlockedDevicesToShareWith) {
+    for (const [userId, userDevices] of Object.entries(devicesByUser)) {
+        for (const deviceInfo of Object.values(userDevices)) {
             currentSlice.push({
                 userId: userId,
-                blockedInfo: blockedInfo,
+                deviceInfo: deviceInfo.device,
             });
         }
 
@@ -473,7 +458,7 @@ MegolmEncryption.prototype._splitBlockedDevices = function(devicesByUser) {
         // server (e.g. only have to send one EDU if a remote user, etc). This
         // does mean that if a user has many devices we may go over the desired
         // limit, but its not a hard limit so that is fine.
-        if (currentSlice.length > maxUsersPerRequest) {
+        if (currentSlice.length > maxDevicesPerRequest) {
             // the current slice is filled up. Start inserting into the next slice
             currentSlice = [];
             mapSlices.push(currentSlice);
@@ -568,7 +553,7 @@ MegolmEncryption.prototype._sendBlockedNotificationsToDevices = async function(
 
     for (const val of userDeviceMap) {
         const userId = val.userId;
-        const blockedInfo = val.blockedInfo;
+        const blockedInfo = val.deviceInfo;
         const deviceInfo = blockedInfo.deviceInfo;
         const deviceId = deviceInfo.deviceId;
 
@@ -643,9 +628,7 @@ MegolmEncryption.prototype.reshareKeyWithDevice = async function(
 
     await olmlib.ensureOlmSessionsForDevices(
         this._olmDevice, this._baseApis, {
-            [userId]: {
-                [device.deviceId]: device,
-            },
+            [userId]: [device],
         },
     );
 
@@ -688,37 +671,44 @@ MegolmEncryption.prototype.reshareKeyWithDevice = async function(
 };
 
 /**
+ * @private
+ *
  * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
  *
+ * @param {object} key the session key as returned by
+ *    OlmDevice.getOutboundGroupSessionKey
+ *
+ * @param {object} payload the base to-device message payload for sharing keys
+ *
  * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
  *    map from userid to list of devices
  *
  * @param {array<object>} errorDevices
  *    array that will be populated with the devices that we can't get an
  *    olm session for
+ *
+ * @param {Number} [otkTimeout] The timeout in milliseconds when requesting
+ *     one-time keys for establishing new olm sessions.
+ *
+ * @param {Array} [failedServers] An array to fill with remote servers that
+ *     failed to respond to one-time-key requests.
  */
 MegolmEncryption.prototype._shareKeyWithDevices = async function(
-    session, devicesByUser, errorDevices,
+    session, key, payload, devicesByUser, errorDevices, otkTimeout, failedServers,
 ) {
-    const key = this._olmDevice.getOutboundGroupSessionKey(session.sessionId);
-    const payload = {
-        type: "m.room_key",
-        content: {
-            algorithm: olmlib.MEGOLM_ALGORITHM,
-            room_id: this._roomId,
-            session_id: session.sessionId,
-            session_key: key.key,
-            chain_index: key.chain_index,
-        },
-    };
-
     const devicemap = await olmlib.ensureOlmSessionsForDevices(
-        this._olmDevice, this._baseApis, devicesByUser,
+        this._olmDevice, this._baseApis, devicesByUser, otkTimeout, failedServers,
     );
 
-    const userDeviceMaps = this._splitUserDeviceMap(
-        session, key.chain_index, devicemap, devicesByUser, errorDevices,
-    );
+    this._getDevicesWithoutSessions(devicemap, devicesByUser, errorDevices);
+
+    await this._shareKeyWithOlmSessions(session, key, payload, devicemap);
+};
+
+MegolmEncryption.prototype._shareKeyWithOlmSessions = async function(
+    session, key, payload, devicemap,
+) {
+    const userDeviceMaps = this._splitDevices(devicemap);
 
     for (let i = 0; i < userDeviceMaps.length; i++) {
         try {
@@ -736,6 +726,52 @@ MegolmEncryption.prototype._shareKeyWithDevices = async function(
     }
 };
 
+/**
+ * Notify devices that we weren't able to create olm sessions.
+ *
+ * @param {module:crypto/algorithms/megolm.OutboundSessionInfo} session
+ *
+ * @param {object} key
+ *
+ * @param {Array<object>} failedDevices the devices that we were unable to
+ *     create olm sessions for, as returned by _shareKeyWithDevices
+ */
+MegolmEncryption.prototype._notifyFailedOlmDevices = async function(
+    session, key, failedDevices,
+) {
+    // mark the devices that failed as "handled" because we don't want to try
+    // to claim a one-time-key for dead devices on every message.
+    for (const {userId, deviceInfo} of failedDevices) {
+        const deviceId = deviceInfo.deviceId;
+
+        session.markSharedWithDevice(
+            userId, deviceId, key.chain_index,
+        );
+    }
+
+    const filteredFailedDevices =
+          await this._olmDevice.filterOutNotifiedErrorDevices(
+              failedDevices,
+          );
+    const blockedMap = {};
+    for (const {userId, deviceInfo} of filteredFailedDevices) {
+        blockedMap[userId] = blockedMap[userId] || {};
+        // we use a similar format to what
+        // olmlib.ensureOlmSessionsForDevices returns, so that
+        // we can use the same function to split
+        blockedMap[userId][deviceInfo.deviceId] = {
+            device: {
+                code: "m.no_olm",
+                reason: WITHHELD_MESSAGES["m.no_olm"],
+                deviceInfo,
+            },
+        };
+    }
+
+    // send the notifications
+    await this._notifyBlockedDevices(session, blockedMap);
+};
+
 /**
  * Notify blocked devices that they have been blocked.
  *
@@ -754,7 +790,7 @@ MegolmEncryption.prototype._notifyBlockedDevices = async function(
         sender_key: this._olmDevice.deviceCurve25519Key,
     };
 
-    const userDeviceMaps = this._splitBlockedDevices(devicesByUser);
+    const userDeviceMaps = this._splitDevices(devicesByUser);
 
     for (let i = 0; i < userDeviceMaps.length; i++) {
         try {
@@ -772,6 +808,38 @@ MegolmEncryption.prototype._notifyBlockedDevices = async function(
     }
 };
 
+/**
+ * Perform any background tasks that can be done before a message is ready to
+ * send, in order to speed up sending of the message.
+ *
+ * @param {module:models/room} room the room the event is in
+ */
+MegolmEncryption.prototype.prepareToEncrypt = function(room) {
+    if (this.encryptionPreparation) {
+        // We're already preparing something, so don't do anything else.
+        // FIXME: check if we need to restart
+        // (https://github.com/matrix-org/matrix-js-sdk/issues/1255)
+        return;
+    }
+
+    logger.debug(`Preparing to encrypt events for ${this._roomId}`);
+
+    this.encryptionPreparation = (async () => {
+        const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
+
+        if (this._crypto.getGlobalErrorOnUnknownDevices()) {
+            // Drop unknown devices for now.  When the message gets sent, we'll
+            // throw an error, but we'll still be prepared to send to the known
+            // devices.
+            this._removeUnknownDevices(devicesInRoom);
+        }
+
+        await this._ensureOutboundSession(devicesInRoom, blocked, true);
+
+        delete this.encryptionPreparation;
+    })();
+};
+
 /**
  * @inheritdoc
  *
@@ -782,38 +850,48 @@ MegolmEncryption.prototype._notifyBlockedDevices = async function(
  * @return {Promise} Promise which resolves to the new event body
  */
 MegolmEncryption.prototype.encryptMessage = async function(room, eventType, content) {
-    const self = this;
     logger.log(`Starting to encrypt event for ${this._roomId}`);
 
+    if (this.encryptionPreparation) {
+        // If we started sending keys, wait for it to be done.
+        // FIXME: check if we need to cancel
+        // (https://github.com/matrix-org/matrix-js-sdk/issues/1255)
+        try {
+            await this.encryptionPreparation;
+        } catch (e) {
+            // ignore any errors -- if the preparation failed, we'll just
+            // restart everything here
+        }
+    }
+
     const [devicesInRoom, blocked] = await this._getDevicesInRoom(room);
 
     // check if any of these devices are not yet known to the user.
     // if so, warn the user so they can verify or ignore.
     if (this._crypto.getGlobalErrorOnUnknownDevices()) {
-        self._checkForUnknownDevices(devicesInRoom);
+        this._checkForUnknownDevices(devicesInRoom);
     }
 
-    const session = await self._ensureOutboundSession(devicesInRoom, blocked);
+    const session = await this._ensureOutboundSession(devicesInRoom, blocked);
     const payloadJson = {
-        room_id: self._roomId,
+        room_id: this._roomId,
         type: eventType,
         content: content,
     };
 
-    const ciphertext = self._olmDevice.encryptGroupMessage(
+    const ciphertext = this._olmDevice.encryptGroupMessage(
         session.sessionId, JSON.stringify(payloadJson),
     );
-
     const encryptedContent = {
         algorithm: olmlib.MEGOLM_ALGORITHM,
-        sender_key: self._olmDevice.deviceCurve25519Key,
+        sender_key: this._olmDevice.deviceCurve25519Key,
         ciphertext: ciphertext,
         session_id: session.sessionId,
         // Include our device ID so that recipients can send us a
         // m.new_device message if they don't have our session key.
         // XXX: Do we still need this now that m.new_device messages
         // no longer exist since #483?
-        device_id: self._deviceId,
+        device_id: this._deviceId,
     };
 
     session.useCount++;
@@ -861,6 +939,27 @@ MegolmEncryption.prototype._checkForUnknownDevices = function(devicesInRoom) {
     }
 };
 
+/**
+ * Remove unknown devices from a set of devices.  The devicesInRoom parameter
+ * will be modified.
+ *
+ * @param {Object} devicesInRoom userId -> {deviceId -> object}
+ *   devices we should shared the session with.
+ */
+MegolmEncryption.prototype._removeUnknownDevices = function(devicesInRoom) {
+    for (const [userId, userDevices] of Object.entries(devicesInRoom)) {
+        for (const [deviceId, device] of Object.entries(userDevices)) {
+            if (device.isUnverified() && !device.isKnown()) {
+                delete userDevices[deviceId];
+            }
+        }
+
+        if (Object.keys(userDevices).length === 0) {
+            delete devicesInRoom[userId];
+        }
+    }
+};
+
 /**
  * Get the list of unblocked devices for all users in the room
  *
@@ -904,8 +1003,10 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
                 continue;
             }
 
+            const deviceTrust = this._crypto.checkDeviceTrust(userId, deviceId);
+
             if (userDevices[deviceId].isBlocked() ||
-                (userDevices[deviceId].isUnverified() && isBlacklisting)
+                (!deviceTrust.isVerified() && isBlacklisting)
             ) {
                 if (!blocked[userId]) {
                     blocked[userId] = {};
@@ -1247,6 +1348,9 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
 
     if (content.code === "m.no_olm") {
         const sender = event.getSender();
+        logger.warn(
+            `${sender}:${senderKey} was unable to establish an olm session with us`,
+        );
         // if the sender says that they haven't been able to establish an olm
         // session, let's proactively establish one
 
@@ -1258,21 +1362,30 @@ MegolmDecryption.prototype.onRoomKeyWithheldEvent = async function(event) {
         if (await this._olmDevice.getSessionIdForDevice(senderKey)) {
             // a session has already been established, so we don't need to
             // create a new one.
+            logger.debug("New session already created.  Not creating a new one.");
             await this._olmDevice.recordSessionProblem(senderKey, "no_olm", true);
             this.retryDecryptionFromSender(senderKey);
             return;
         }
-        const device = this._crypto._deviceList.getDeviceByIdentityKey(
+        let device = this._crypto._deviceList.getDeviceByIdentityKey(
             content.algorithm, senderKey,
         );
         if (!device) {
-            logger.info(
-                "Couldn't find device for identity key " + senderKey +
-                    ": not establishing session",
+            // if we don't know about the device, fetch the user's devices again
+            // and retry before giving up
+            await this._crypto.downloadKeys([sender], false);
+            device = this._crypto._deviceList.getDeviceByIdentityKey(
+                content.algorithm, senderKey,
             );
-            await this._olmDevice.recordSessionProblem(senderKey, "no_olm", false);
-            this.retryDecryptionFromSender(senderKey);
-            return;
+            if (!device) {
+                logger.info(
+                    "Couldn't find device for identity key " + senderKey +
+                        ": not establishing session",
+                );
+                await this._olmDevice.recordSessionProblem(senderKey, "no_olm", false);
+                this.retryDecryptionFromSender(senderKey);
+                return;
+            }
         }
         await olmlib.ensureOlmSessionsForDevices(
             this._olmDevice, this._baseApis, {[sender]: [device]}, false,
@@ -1481,7 +1594,6 @@ MegolmDecryption.prototype._retryDecryption = async function(senderKey, sessionI
 
 MegolmDecryption.prototype.retryDecryptionFromSender = async function(senderKey) {
     const senderPendingEvents = this._pendingEvents[senderKey];
-    logger.warn(senderPendingEvents);
     if (!senderPendingEvents) {
         return true;
     }
@@ -1491,7 +1603,6 @@ MegolmDecryption.prototype.retryDecryptionFromSender = async function(senderKey)
     await Promise.all([...senderPendingEvents].map(async ([_sessionId, pending]) => {
         await Promise.all([...pending].map(async (ev) => {
             try {
-                logger.warn(ev.getId());
                 await ev.attemptDecryption(this._crypto);
             } catch (e) {
                 // don't die if something goes wrong

src/crypto/index.js

@@ -37,8 +37,9 @@ import {
     CrossSigningLevel,
     DeviceTrustLevel,
     UserTrustLevel,
+    createCryptoStoreCacheCallbacks,
 } from './CrossSigning';
-import {SECRET_STORAGE_ALGORITHM_V1, SecretStorage} from './SecretStorage';
+import {SECRET_STORAGE_ALGORITHM_V1_AES, SecretStorage} from './SecretStorage';
 import {OutgoingRoomKeyRequestManager} from './OutgoingRoomKeyRequestManager';
 import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
 import {
@@ -54,6 +55,7 @@ import {InRoomChannel, InRoomRequests} from "./verification/request/InRoomChanne
 import {ToDeviceChannel, ToDeviceRequests} from "./verification/request/ToDeviceChannel";
 import * as httpApi from "../http-api";
 import {IllegalMethod} from "./verification/IllegalMethod";
+import {KeySignatureUploadError} from "../errors";
 
 const DeviceVerification = DeviceInfo.DeviceVerification;
 
@@ -118,6 +120,8 @@ export function Crypto(baseApis, sessionStore, userId, deviceId,
     this._onDeviceListUserCrossSigningUpdated =
         this._onDeviceListUserCrossSigningUpdated.bind(this);
 
+    this._trustCrossSignedDevices = true;
+
     this._reEmitter = new ReEmitter(this);
     this._baseApis = baseApis;
     this._sessionStore = sessionStore;
@@ -220,8 +224,13 @@ export function Crypto(baseApis, sessionStore, userId, deviceId,
     this._inRoomVerificationRequests = new InRoomRequests();
 
     const cryptoCallbacks = this._baseApis._cryptoCallbacks || {};
+    const cacheCallbacks = createCryptoStoreCacheCallbacks(cryptoStore);
 
-    this._crossSigningInfo = new CrossSigningInfo(userId, cryptoCallbacks);
+    this._crossSigningInfo = new CrossSigningInfo(
+        userId,
+        cryptoCallbacks,
+        cacheCallbacks,
+    );
 
     this._secretStorage = new SecretStorage(
         baseApis, cryptoCallbacks, this._crossSigningInfo,
@@ -312,6 +321,47 @@ Crypto.prototype.init = async function(opts) {
     this._checkAndStartKeyBackup();
 };
 
+/**
+ * Whether to trust a others users signatures of their devices.
+ * If false, devices will only be considered 'verified' if we have
+ * verified that device individually (effectively disabling cross-signing).
+ *
+ * Default: true
+ *
+ * @return {bool} True if trusting cross-signed devices
+ */
+Crypto.prototype.getCryptoTrustCrossSignedDevices = function() {
+    return this._trustCrossSignedDevices;
+};
+
+/**
+ * See getCryptoTrustCrossSignedDevices
+
+ * This may be set before initCrypto() is called to ensure no races occur.
+ *
+ * @param {bool} val True to trust cross-signed devices
+ */
+Crypto.prototype.setCryptoTrustCrossSignedDevices = function(val) {
+    this._trustCrossSignedDevices = val;
+
+    for (const userId of this._deviceList.getKnownUserIds()) {
+        const devices = this._deviceList.getRawStoredDevicesForUser(userId);
+        for (const deviceId of Object.keys(devices)) {
+            const deviceTrust = this.checkDeviceTrust(userId, deviceId);
+            // If the device is locally verified then isVerified() is always true,
+            // so this will only have caused the value to change if the device is
+            // cross-signing verified but not locally verified
+            if (
+                !deviceTrust.isLocallyVerified() &&
+                deviceTrust.isCrossSigningVerified()
+            ) {
+                const deviceObj = this._deviceList.getStoredDevice(userId, deviceId);
+                this.emit("deviceVerificationChanged", userId, deviceId, deviceObj);
+            }
+        }
+    }
+};
+
 /**
  * Create a recovery key from a user-supplied passphrase.
  *
@@ -345,6 +395,37 @@ Crypto.prototype.createRecoveryKeyFromPassphrase = async function(password) {
     }
 };
 
+/**
+ * Checks whether cross signing:
+ * - is enabled on this account
+ * - is trusted by this device
+ * - has private keys stored in secret storage
+ * and that the account has a secret storage key
+ *
+ * If this function returns false, bootstrapSecretStorage() can be used
+ * to fix things such that it returns true. That is to say, after
+ * bootstrapSecretStorage() completes sucessfully, this function should
+ * return true.
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @return {bool} True if cross-signing is ready to be used on this device
+ */
+Crypto.prototype.isCrossSigningReady = async function() {
+    const publicKeysOnDevice = this._crossSigningInfo.getId();
+    const privateKeysInStorage = await this._crossSigningInfo.isStoredInSecretStorage(
+        this._secretStorage,
+    );
+    const secretStorageKeyInAccount = await this._secretStorage.hasKey();
+
+    return (
+        publicKeysOnDevice &&
+        privateKeysInStorage &&
+        secretStorageKeyInAccount
+    );
+};
+
+
 /**
  * Bootstrap Secure Secret Storage if needed by creating a default key and
  * signing it with the cross-signing master key. If everything is already set
@@ -364,6 +445,9 @@ Crypto.prototype.createRecoveryKeyFromPassphrase = async function(password) {
  * created and the private key stored in the new SSSS store. Ignored if keyBackupInfo
  * is supplied.
  * @param {bool} [opts.setupNewSecretStorage] Optional. Reset even if keys already exist.
+ * @param {func} [opts.getKeyBackupPassphrase] Optional. Function called to get the user's
+ *     current key backup passphrase. Should return a promise that resolves with a Buffer
+ *     containing the key, or rejects if the key cannot be obtained.
  * Returns:
  *     {Promise} A promise which resolves to key creation data for
  *     SecretStorage#addKey: an object with `passphrase` and/or `pubkey` fields.
@@ -374,6 +458,7 @@ Crypto.prototype.bootstrapSecretStorage = async function({
     keyBackupInfo,
     setupNewKeyBackup,
     setupNewSecretStorage,
+    getKeyBackupPassphrase,
 } = {}) {
     logger.log("Bootstrapping Secure Secret Storage");
 
@@ -388,10 +473,82 @@ Crypto.prototype.bootstrapSecretStorage = async function({
     // use temporary callbacks to weave them through the various APIs.
     const appCallbacks = Object.assign({}, this._baseApis._cryptoCallbacks);
 
+    // the ID of the new SSSS key, if we create one
+    let newKeyId = null;
+
+    // cache SSSS keys so that we don't need to constantly pester the user about it
+    const ssssKeys = {};
+
+    this._baseApis._cryptoCallbacks.getSecretStorageKey =
+        async ({keys}, name) => {
+            // if we already have a key that works, return it
+            for (const keyId of Object.keys(keys)) {
+                if (ssssKeys[keyId]) {
+                    return [keyId, ssssKeys[keyId]];
+                }
+            }
+
+            // otherwise, prompt the user and cache it
+            const key = await appCallbacks.getSecretStorageKey({keys}, name);
+            if (key) {
+                const [keyId, keyData] = key;
+                ssssKeys[keyId] = keyData;
+            }
+            return key;
+        };
+
     try {
-        const inStorage = !setupNewSecretStorage &&
-            await this._crossSigningInfo.isStoredInSecretStorage(this._secretStorage);
-        if (!this._crossSigningInfo.getId() || !inStorage) {
+        const decryptionKeys =
+              await this._crossSigningInfo.isStoredInSecretStorage(this._secretStorage);
+        const inStorage = !setupNewSecretStorage && decryptionKeys;
+        if (decryptionKeys && !(Object.values(decryptionKeys).some(
+            info => info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES,
+        ))) {
+            // we already have cross-signing keys, but they're encrypted using
+            // the old algorithm
+            logger.log("Switching to symmetric");
+            const keys = {};
+            // fetch the cross-signing private keys (needed to sign the new
+            // SSSS key).  We store the cross-signing keys, and temporarily set
+            // a callback so that when the private key is needed while setting
+            // things up, we can provide it.
+            this._baseApis._cryptoCallbacks.getCrossSigningKey =
+                name => crossSigningPrivateKeys[name];
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                const secretName = `m.cross_signing.${type}`;
+                const secret = await this.getSecret(secretName);
+                keys[type] = secret;
+                crossSigningPrivateKeys[type] = olmlib.decodeBase64(secret);
+            }
+            await this.checkOwnCrossSigningTrust();
+            const opts = {};
+            let oldKeyId = null;
+            for (const [keyId, keyInfo] of Object.entries(decryptionKeys)) {
+                // See if the old key was generated from a passphrase.  If
+                // yes, use the same settings.
+                if (keyId in ssssKeys) {
+                    oldKeyId = keyId;
+                    if (keyInfo.passphrase) {
+                        opts.passphrase = keyInfo.passphrase;
+                    }
+                    break;
+                }
+            }
+            // create new symmetric SSSS key and set it as default
+            newKeyId = await this.addSecretStorageKey(
+                SECRET_STORAGE_ALGORITHM_V1_AES, opts,
+            );
+            if (oldKeyId) {
+                ssssKeys[newKeyId] = ssssKeys[oldKeyId];
+            }
+            await this.setDefaultSecretStorageKeyId(newKeyId);
+            // re-encrypt all the keys with the new key
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                const secretName = `m.cross_signing.${type}`;
+                await this.storeSecret(secretName, keys[type], [newKeyId]);
+            }
+        } else if (!this._crossSigningInfo.getId() || !inStorage) {
+            // create new cross-signing keys if necessary.
             logger.log(
                 "Cross-signing public and/or private keys not found, " +
                 "checking secret storage for private keys",
@@ -417,40 +574,50 @@ Crypto.prototype.bootstrapSecretStorage = async function({
             logger.log("Cross signing keys are present in secret storage");
         }
 
-        // Check if Secure Secret Storage has a default key. If we don't have one, create
-        // the default key (which will also be signed by the cross-signing master key).
-        if (setupNewSecretStorage || !await this.hasSecretStorageKey()) {
-            let newKeyId;
+        // Check if we need to create a new secret storage key
+        // - we're resetting secret storage
+        // - we don't have a default secret storage key yet
+        // - our default secret storage key is using an older algorithm
+        // We will also run this part if we created a new secret storage key
+        // above, so that we can (re-)encrypt the backup with it.
+        const defaultSSSSKey = await this.getSecretStorageKey();
+        if (setupNewSecretStorage || newKeyId || !defaultSSSSKey
+            || defaultSSSSKey[1].algorithm !== SECRET_STORAGE_ALGORITHM_V1_AES) {
             if (keyBackupInfo) {
+                // if we already have a backup key, use the same key as the
+                // secret storage key
                 logger.log("Secret storage default key not found, using key backup key");
-                const opts = {
-                    pubkey: keyBackupInfo.auth_data.public_key,
-                };
 
-                if (
-                    keyBackupInfo.auth_data.private_key_salt &&
-                    keyBackupInfo.auth_data.private_key_iterations
-                ) {
-                    opts.passphrase = {
-                        algorithm: "m.pbkdf2",
-                        iterations: keyBackupInfo.auth_data.private_key_iterations,
-                        salt: keyBackupInfo.auth_data.private_key_salt,
-                    };
+                const backupKey = await getKeyBackupPassphrase();
+
+                if (!newKeyId) {
+                    const opts = {};
+
+                    if (
+                        keyBackupInfo.auth_data.private_key_salt &&
+                        keyBackupInfo.auth_data.private_key_iterations
+                    ) {
+                        opts.passphrase = {
+                            algorithm: "m.pbkdf2",
+                            iterations: keyBackupInfo.auth_data.private_key_iterations,
+                            salt: keyBackupInfo.auth_data.private_key_salt,
+                            bits: 256,
+                        };
+                    }
+
+                    newKeyId = await this.addSecretStorageKey(
+                        SECRET_STORAGE_ALGORITHM_V1_AES, opts,
+                    );
+                    this.setDefaultSecretStorageKeyId(newKeyId);
+                    // use the backup key as the new ssss key
+                    ssssKeys[newKeyId] = backupKey;
                 }
 
-                newKeyId = await this.addSecretStorageKey(
-                    SECRET_STORAGE_ALGORITHM_V1, opts,
-                );
-
-                // Add an entry for the backup key in SSSS as a 'passthrough' key
-                // (ie. the secret is the key itself).
-                this._secretStorage.storePassthrough('m.megolm_backup.v1', newKeyId);
-
                 // if this key backup is trusted, sign it with the cross signing key
                 // so the key backup can be trusted via cross-signing.
                 const backupSigStatus = await this.checkKeyBackup(keyBackupInfo);
                 if (backupSigStatus.trustInfo.usable) {
-                    console.log("Adding cross signing signature to key backup");
+                    logger.log("Adding cross signing signature to key backup");
                     await this._crossSigningInfo.signObject(
                         keyBackupInfo.auth_data, "master",
                     );
@@ -459,20 +626,32 @@ Crypto.prototype.bootstrapSecretStorage = async function({
                         undefined, keyBackupInfo,
                         {prefix: httpApi.PREFIX_UNSTABLE},
                     );
+                    await this.storeSecret(
+                        "m.megolm_backup.v1", olmlib.encodeBase64(backupKey), [newKeyId],
+                    );
                 } else {
-                    console.log(
+                    logger.log(
                         "Key backup is NOT TRUSTED: NOT adding cross signing signature",
                     );
                 }
             } else {
-                logger.log("Secret storage default key not found, creating new key");
-                const keyOptions = await createSecretStorageKey();
-                newKeyId = await this.addSecretStorageKey(
-                    SECRET_STORAGE_ALGORITHM_V1,
-                    keyOptions,
-                );
+                if (!newKeyId) {
+                    logger.log("Secret storage default key not found, creating new key");
+                    const keyOptions = await createSecretStorageKey();
+                    newKeyId = await this.addSecretStorageKey(
+                        SECRET_STORAGE_ALGORITHM_V1_AES,
+                        keyOptions,
+                    );
+                    await this.setDefaultSecretStorageKeyId(newKeyId);
+                }
+                if (await this.isSecretStored("m.megolm_backup.v1")) {
+                    // we created a new SSSS, and we previously encrypted the
+                    // backup key with the old SSSS key, so re-encrypt with the
+                    // new key
+                    const backupKey = await this.getSecret("m.megolm_backup.v1");
+                    await this.storeSecret("m.megolm_backup.v1", backupKey, [newKeyId]);
+                }
             }
-            await this.setDefaultSecretStorageKeyId(newKeyId);
         } else {
             logger.log("Have secret storage key");
         }
@@ -498,6 +677,15 @@ Crypto.prototype.bootstrapSecretStorage = async function({
             }
         }
 
+        // Call `getCrossSigningKey` for side effect of caching private keys for
+        // future gossiping to other devices if enabled via app level callbacks.
+        if (this._crossSigningInfo._cacheCallbacks) {
+            for (const type of ["self_signing", "user_signing"]) {
+                logger.log(`Cache ${type} cross-signing private key locally`);
+                await this._crossSigningInfo.getCrossSigningKey(type);
+            }
+        }
+
         if (setupNewKeyBackup && !keyBackupInfo) {
             const info = await this._baseApis.prepareKeyBackupVersion(
                 null /* random key */,
@@ -528,6 +716,14 @@ Crypto.prototype.hasSecretStorageKey = function(keyID) {
     return this._secretStorage.hasKey(keyID);
 };
 
+Crypto.prototype.secretStorageKeyNeedsUpgrade = function(keyID) {
+    return this._secretStorage.keyNeedsUpgrade(keyID);
+};
+
+Crypto.prototype.getSecretStorageKey = function(keyID) {
+    return this._secretStorage.getKey(keyID);
+};
+
 Crypto.prototype.storeSecret = function(name, secret, keys) {
     return this._secretStorage.store(name, secret, keys);
 };
@@ -576,6 +772,41 @@ Crypto.prototype.checkSecretStoragePrivateKey = function(privateKey, expectedPub
     }
 };
 
+/**
+ * Fetches the backup private key, if cached
+ * @returns {Promise} the key, if any, or null
+ */
+Crypto.prototype.getSessionBackupPrivateKey = async function() {
+    return new Promise((resolve) => {
+        this._cryptoStore.doTxn(
+            'readonly',
+            [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                this._cryptoStore.getSecretStorePrivateKey(
+                    txn,
+                    resolve,
+                    "m.megolm_backup.v1",
+                );
+            },
+        );
+    });
+};
+
+/**
+ * Stores the session backup key to the cache
+ * @param {Uint8Array} key the private key
+ * @returns {Promise} so you can catch failures
+ */
+Crypto.prototype.storeSessionBackupPrivateKey = async function(key) {
+    return this._cryptoStore.doTxn(
+        'readwrite',
+        [IndexedDBCryptoStore.STORE_ACCOUNT],
+        (txn) => {
+            this._cryptoStore.storeSecretStorePrivateKey(txn, "m.megolm_backup.v1", key);
+        },
+    );
+};
+
 /**
  * Checks that a given cross-signing private key matches a given public key.
  * This can be used by the getCrossSigningKey callback to verify that the
@@ -655,49 +886,84 @@ Crypto.prototype.resetCrossSigningKeys = async function(level, {
  * verifications, etc.
  */
 Crypto.prototype._afterCrossSigningLocalKeyChange = async function() {
+    logger.info("Starting cross-signing key change post-processing");
+
     // sign the current device with the new key, and upload to the server
     const device = this._deviceList.getStoredDevice(this._userId, this._deviceId);
     const signedDevice = await this._crossSigningInfo.signDevice(this._userId, device);
-    await this._baseApis.uploadKeySignatures({
-        [this._userId]: {
-            [this._deviceId]: signedDevice,
-        },
-    });
+    logger.info(`Starting background key sig upload for ${this._deviceId}`);
 
-    // check all users for signatures
-    // FIXME: do this in batches
-    const users = {};
-    for (const [userId, crossSigningInfo]
-         of Object.entries(this._deviceList._crossSigningInfo)) {
-        const upgradeInfo = await this._checkForDeviceVerificationUpgrade(
-            userId, CrossSigningInfo.fromStorage(crossSigningInfo, userId),
-        );
-        if (upgradeInfo) {
-            users[userId] = upgradeInfo;
-        }
-    }
+    const upload = ({ shouldEmit }) => {
+        return this._baseApis.uploadKeySignatures({
+            [this._userId]: {
+                [this._deviceId]: signedDevice,
+            },
+        }).then((response) => {
+            const { failures } = response || {};
+            if (Object.keys(failures || []).length > 0) {
+                if (shouldEmit) {
+                    this._baseApis.emit(
+                        "crypto.keySignatureUploadFailure",
+                        failures,
+                        "_afterCrossSigningLocalKeyChange",
+                        upload, // continuation
+                    );
+                }
+                throw new KeySignatureUploadError("Key upload failed", { failures });
+            }
+            logger.info(`Finished background key sig upload for ${this._deviceId}`);
+        }).catch(e => {
+            logger.error(
+                `Error during background key sig upload for ${this._deviceId}`,
+                e,
+            );
+        });
+    };
+    upload({ shouldEmit: true });
 
     const shouldUpgradeCb = (
         this._baseApis._cryptoCallbacks.shouldUpgradeDeviceVerifications
     );
-    if (Object.keys(users).length > 0 && shouldUpgradeCb) {
-        try {
-            const usersToUpgrade = await shouldUpgradeCb({users: users});
-            if (usersToUpgrade) {
-                for (const userId of usersToUpgrade) {
-                    if (userId in users) {
-                        await this._baseApis.setDeviceVerified(
-                            userId, users[userId].crossSigningInfo.getId(),
-                        );
+    if (shouldUpgradeCb) {
+        logger.info("Starting device verification upgrade");
+
+        // Check all users for signatures if upgrade callback present
+        // FIXME: do this in batches
+        const users = {};
+        for (const [userId, crossSigningInfo]
+            of Object.entries(this._deviceList._crossSigningInfo)) {
+            const upgradeInfo = await this._checkForDeviceVerificationUpgrade(
+                userId, CrossSigningInfo.fromStorage(crossSigningInfo, userId),
+            );
+            if (upgradeInfo) {
+                users[userId] = upgradeInfo;
+            }
+        }
+
+        if (Object.keys(users).length > 0) {
+            logger.info(`Found ${Object.keys(users).length} verif users to upgrade`);
+            try {
+                const usersToUpgrade = await shouldUpgradeCb({ users: users });
+                if (usersToUpgrade) {
+                    for (const userId of usersToUpgrade) {
+                        if (userId in users) {
+                            await this._baseApis.setDeviceVerified(
+                                userId, users[userId].crossSigningInfo.getId(),
+                            );
+                        }
                     }
                 }
+            } catch (e) {
+                logger.log(
+                    "shouldUpgradeDeviceVerifications threw an error: not upgrading", e,
+                );
             }
-        } catch (e) {
-            logger.log(
-                "shouldUpgradeDeviceVerifications threw an error: not upgrading", e,
-            );
         }
+
+        logger.info("Finished device verification upgrade");
     }
+
+    logger.info("Finished cross-signing key change post-processing");
 };
 
 /**
@@ -808,15 +1074,30 @@ Crypto.prototype.checkUserTrust = function(userId) {
  */
 Crypto.prototype.checkDeviceTrust = function(userId, deviceId) {
     const device = this._deviceList.getStoredDevice(userId, deviceId);
-    const trustedLocally = device && device.isVerified();
+    return this._checkDeviceInfoTrust(userId, device);
+};
+
+/**
+ * Check whether a given deviceinfo is trusted.
+ *
+ * @param {string} userId The ID of the user whose devices is to be checked.
+ * @param {module:crypto/deviceinfo?} device The device info object to check
+ *
+ * @returns {DeviceTrustLevel}
+ */
+Crypto.prototype._checkDeviceInfoTrust = function(userId, device) {
+    const trustedLocally = !!(device && device.isVerified());
 
     const userCrossSigning = this._deviceList.getStoredCrossSigningForUser(userId);
     if (device && userCrossSigning) {
+        // The _trustCrossSignedDevices only affects trust of other people's cross-signing
+        // signatures
+        const trustCrossSig = this._trustCrossSignedDevices || userId === this._userId;
         return this._crossSigningInfo.checkDeviceTrust(
-            userCrossSigning, device, trustedLocally,
+            userCrossSigning, device, trustedLocally, trustCrossSig,
         );
     } else {
-        return new DeviceTrustLevel(false, false, trustedLocally);
+        return new DeviceTrustLevel(false, false, trustedLocally, false);
     }
 };
 
@@ -918,13 +1199,52 @@ Crypto.prototype.checkOwnCrossSigningTrust = async function() {
     }
 
     if (masterChanged) {
-        await this._signObject(this._crossSigningInfo.keys.master);
-        keySignatures[this._crossSigningInfo.getId()]
-            = this._crossSigningInfo.keys.master;
+        const masterKey = this._crossSigningInfo.keys.master;
+        await this._signObject(masterKey);
+        const deviceSig = masterKey.signatures[this._userId]["ed25519:" + this._deviceId];
+        // Include only the _new_ device signature in the upload.
+        // We may have existing signatures from deleted devices, which will cause
+        // the entire upload to fail.
+        keySignatures[this._crossSigningInfo.getId()] = Object.assign(
+            {},
+            masterKey,
+            {
+                signatures: {
+                    [this._userId]: {
+                        ["ed25519:" + this._deviceId]: deviceSig,
+                    },
+                },
+            },
+        );
     }
 
-    if (Object.keys(keySignatures).length) {
-        await this._baseApis.uploadKeySignatures({[this._userId]: keySignatures});
+    const keysToUpload = Object.keys(keySignatures);
+    if (keysToUpload.length) {
+        const upload = ({ shouldEmit }) => {
+            logger.info(`Starting background key sig upload for ${keysToUpload}`);
+            return this._baseApis.uploadKeySignatures({ [this._userId]: keySignatures })
+            .then((response) => {
+                const { failures } = response || {};
+                logger.info(`Finished background key sig upload for ${keysToUpload}`);
+                if (Object.keys(failures || []).length > 0) {
+                    if (shouldEmit) {
+                        this._baseApis.emit(
+                            "crypto.keySignatureUploadFailure",
+                            failures,
+                            "checkOwnCrossSigningTrust",
+                            upload,
+                        );
+                    }
+                    throw new KeySignatureUploadError("Key upload failed", { failures });
+                }
+            }).catch(e => {
+                logger.error(
+                    `Error during background key sig upload for ${keysToUpload}`,
+                    e,
+                );
+            });
+        };
+        upload({ shouldEmit: true });
     }
 
     this.emit("userTrustStatusChanged", userId, this.checkUserTrust(userId));
@@ -962,16 +1282,21 @@ Crypto.prototype._storeTrustedSelfKeys = async function(keys) {
  * @param {string} userId the user ID whose key should be checked
  */
 Crypto.prototype._checkDeviceVerifications = async function(userId) {
+    const shouldUpgradeCb = (
+        this._baseApis._cryptoCallbacks.shouldUpgradeDeviceVerifications
+    );
+    if (!shouldUpgradeCb) {
+        // Upgrading skipped when callback is not present.
+        return;
+    }
+    logger.info(`Starting device verification upgrade for ${userId}`);
     if (this._crossSigningInfo.keys.user_signing) {
         const crossSigningInfo = this._deviceList.getStoredCrossSigningForUser(userId);
         if (crossSigningInfo) {
             const upgradeInfo = await this._checkForDeviceVerificationUpgrade(
                 userId, crossSigningInfo,
             );
-            const shouldUpgradeCb = (
-                this._baseApis._cryptoCallbacks.shouldUpgradeDeviceVerifications
-            );
-            if (upgradeInfo && shouldUpgradeCb) {
+            if (upgradeInfo) {
                 const usersToUpgrade = await shouldUpgradeCb({
                     users: {
                         [userId]: upgradeInfo,
@@ -985,6 +1310,7 @@ Crypto.prototype._checkDeviceVerifications = async function(userId) {
             }
         }
     }
+    logger.info(`Finished device verification upgrade for ${userId}`);
 };
 
 /**
@@ -1243,6 +1569,15 @@ Crypto.prototype.getDeviceEd25519Key = function() {
     return this._olmDevice.deviceEd25519Key;
 };
 
+/**
+ * Get the Curve25519 key for this device
+ *
+ * @return {string} base64-encoded curve25519 key.
+ */
+Crypto.prototype.getDeviceCurve25519Key = function() {
+    return this._olmDevice.deviceCurve25519Key;
+};
+
 /**
  * Set the global override for whether the client should ever send encrypted
  * messages to unverified devices.  This provides the default for rooms which
@@ -1550,12 +1885,33 @@ Crypto.prototype.setDeviceVerification = async function(
             );
             const device = await this._crossSigningInfo.signUser(xsk);
             if (device) {
-                logger.info("Uploading signature for " + userId + "...");
-                await this._baseApis.uploadKeySignatures({
-                    [userId]: {
-                        [deviceId]: device,
-                    },
-                });
+                const upload = async ({ shouldEmit }) => {
+                    logger.info("Uploading signature for " + userId + "...");
+                    const response = await this._baseApis.uploadKeySignatures({
+                        [userId]: {
+                            [deviceId]: device,
+                        },
+                    });
+                    const { failures } = response || {};
+                    if (Object.keys(failures || []).length > 0) {
+                        if (shouldEmit) {
+                            this._baseApis.emit(
+                                "crypto.keySignatureUploadFailure",
+                                failures,
+                                "setDeviceVerification",
+                                upload,
+                            );
+                        }
+                        /* Throwing here causes the process to be cancelled and the other
+                        * user to be notified */
+                        throw new KeySignatureUploadError(
+                            "Key upload failed",
+                            { failures },
+                        );
+                    }
+                };
+                await upload({ shouldEmit: true });
+
                 // This will emit events when it comes back down the sync
                 // (we could do local echo to speed things up)
             }
@@ -1604,12 +1960,27 @@ Crypto.prototype.setDeviceVerification = async function(
             userId, DeviceInfo.fromStorage(dev, deviceId),
         );
         if (device) {
-            logger.info("Uploading signature for " + deviceId);
-            await this._baseApis.uploadKeySignatures({
-                [userId]: {
-                    [deviceId]: device,
-                },
-            });
+            const upload = async ({shouldEmit}) => {
+                logger.info("Uploading signature for " + deviceId);
+                const response = await this._baseApis.uploadKeySignatures({
+                    [userId]: {
+                        [deviceId]: device,
+                    },
+                });
+                const { failures } = response || {};
+                if (Object.keys(failures || []).length > 0) {
+                    if (shouldEmit) {
+                        this._baseApis.emit(
+                            "crypto.keySignatureUploadFailure",
+                            failures,
+                            "setDeviceVerification",
+                            upload, // continuation
+                        );
+                    }
+                    throw new KeySignatureUploadError("Key upload failed", { failures });
+                }
+            };
+            await upload({shouldEmit: true});
             // XXX: we'll need to wait for the device list to be updated
         }
     }
@@ -1619,7 +1990,16 @@ Crypto.prototype.setDeviceVerification = async function(
     return deviceObj;
 };
 
+Crypto.prototype.findVerificationRequestDMInProgress = function(roomId) {
+    return this._inRoomVerificationRequests.findRequestInProgress(roomId);
+};
+
 Crypto.prototype.requestVerificationDM = function(userId, roomId) {
+    const existingRequest = this._inRoomVerificationRequests.
+        findRequestInProgress(roomId);
+    if (existingRequest) {
+        return Promise.resolve(existingRequest);
+    }
     const channel = new InRoomChannel(this._baseApis, roomId, userId);
     return this._requestVerificationWithChannel(
         userId,
@@ -1632,6 +2012,11 @@ Crypto.prototype.requestVerification = function(userId, devices) {
     if (!devices) {
         devices = Object.keys(this._deviceList.getRawStoredDevicesForUser(userId));
     }
+    const existingRequest = this._toDeviceVerificationRequests
+        .findRequestInProgress(userId, devices);
+    if (existingRequest) {
+        return Promise.resolve(existingRequest);
+    }
     const channel = new ToDeviceChannel(this._baseApis, userId, devices);
     return this._requestVerificationWithChannel(
         userId,
@@ -1682,6 +2067,27 @@ Crypto.prototype.beginKeyVerification = function(
     return request.beginKeyVerification(method, {userId, deviceId});
 };
 
+Crypto.prototype.legacyDeviceVerification = async function(
+    userId, deviceId, method,
+) {
+    const transactionId = ToDeviceChannel.makeTransactionId();
+    const channel = new ToDeviceChannel(
+        this._baseApis, userId, [deviceId], transactionId, deviceId);
+    const request = new VerificationRequest(
+        channel, this._verificationMethods, this._baseApis);
+    this._toDeviceVerificationRequests.setRequestBySenderAndTxnId(
+        userId, transactionId, request);
+    const verifier = request.beginKeyVerification(method, {userId, deviceId});
+    // either reject by an error from verify() while sending .start
+    // or resolve when the request receives the
+    // local (fake remote) echo for sending the .start event
+    await Promise.race([
+        verifier.verify(),
+        request.waitFor(r => r.started),
+    ]);
+    return request;
+};
+
 
 /**
  * Get information on the active olm sessions with a user
@@ -2099,14 +2505,18 @@ Crypto.prototype._backupPendingKeys = async function(limit) {
         const forwardedCount =
               (sessionData.forwarding_curve25519_key_chain || []).length;
 
+        const userId = this._deviceList.getUserByIdentityKey(
+            olmlib.MEGOLM_ALGORITHM, session.senderKey,
+        );
         const device = this._deviceList.getDeviceByIdentityKey(
             olmlib.MEGOLM_ALGORITHM, session.senderKey,
         );
+        const verified = this._checkDeviceInfoTrust(userId, device).isVerified();
 
         data[roomId]['sessions'][session.sessionId] = {
             first_message_index: firstKnownIndex,
             forwarded_count: forwardedCount,
-            is_verified: !!(device && device.isVerified()),
+            is_verified: verified,
             session_data: encrypted,
         };
     }
@@ -2179,6 +2589,20 @@ Crypto.prototype.flagAllGroupSessionsForBackup = async function() {
     return remaining;
 };
 
+/**
+ * Perform any background tasks that can be done before a message is ready to
+ * send, in order to speed up sending of the message.
+ *
+ * @param {module:models/room} room the room the event is in
+ */
+Crypto.prototype.prepareToEncrypt = function(room) {
+    const roomId = room.roomId;
+    const alg = this._roomEncryptors[roomId];
+    if (alg) {
+        alg.prepareToEncrypt(room);
+    }
+};
+
 /* eslint-disable valid-jsdoc */    //https://github.com/eslint/eslint/issues/7307
 /**
  * Encrypt an event according to the configuration of the room.
@@ -2638,12 +3062,7 @@ Crypto.prototype._handleVerificationEvent = async function(
     }
     event.setVerificationRequest(request);
     try {
-        const hadVerifier = !!request.verifier;
         await request.channel.handleEvent(event, request, isLiveEvent);
-        // emit start event when verifier got set
-        if (!hadVerifier && request.verifier) {
-            this._baseApis.emit("crypto.verification.start", request.verifier);
-        }
     } catch (err) {
         logger.error("error while handling verification event: " + err.message);
     }
@@ -2700,15 +3119,21 @@ Crypto.prototype._onToDeviceBadEncrypted = async function(event) {
     // on a current session.
     // Note that an undecryptable message from another device could easily be spoofed -
     // is there anything we can do to mitigate this?
-    const device = this._deviceList.getDeviceByIdentityKey(algorithm, deviceKey);
+    let device = this._deviceList.getDeviceByIdentityKey(algorithm, deviceKey);
     if (!device) {
-        logger.info(
-            "Couldn't find device for identity key " + deviceKey +
-            ": not re-establishing session",
-        );
-        await this._olmDevice.recordSessionProblem(deviceKey, "wedged", false);
-        retryDecryption();
-        return;
+        // if we don't know about the device, fetch the user's devices again
+        // and retry before giving up
+        await this.downloadKeys([sender], false);
+        device = this._deviceList.getDeviceByIdentityKey(algorithm, deviceKey);
+        if (!device) {
+            logger.info(
+                "Couldn't find device for identity key " + deviceKey +
+                    ": not re-establishing session",
+            );
+            await this._olmDevice.recordSessionProblem(deviceKey, "wedged", false);
+            retryDecryption();
+            return;
+        }
     }
     const devicesByUser = {};
     devicesByUser[sender] = [device];
@@ -2935,9 +3360,8 @@ Crypto.prototype._processReceivedRoomKeyRequest = async function(req) {
         decryptor.shareKeysWithDevice(req);
     };
 
-    // if the device is is verified already, share the keys
-    const device = this._deviceList.getStoredDevice(userId, deviceId);
-    if (device && device.isVerified()) {
+    // if the device is verified already, share the keys
+    if (this.checkDeviceTrust(userId, deviceId).isVerified()) {
         logger.log('device is already verified: sharing keys');
         req.share();
         return;

src/crypto/key_passphrase.js

@@ -19,6 +19,8 @@ import {randomString} from '../randomstring';
 
 const DEFAULT_ITERATIONS = 500000;
 
+const DEFAULT_BITSIZE = 256;
+
 export async function keyFromAuthData(authData, password) {
     if (!global.Olm) {
         throw new Error("Olm is not available");
@@ -34,6 +36,7 @@ export async function keyFromAuthData(authData, password) {
     return await deriveKey(
         password, authData.private_key_salt,
         authData.private_key_iterations,
+        authData.private_key_bits || DEFAULT_BITSIZE,
     );
 }
 
@@ -44,12 +47,12 @@ export async function keyFromPassphrase(password) {
 
     const salt = randomString(32);
 
-    const key = await deriveKey(password, salt, DEFAULT_ITERATIONS);
+    const key = await deriveKey(password, salt, DEFAULT_ITERATIONS, DEFAULT_BITSIZE);
 
     return { key, salt, iterations: DEFAULT_ITERATIONS };
 }
 
-export async function deriveKey(password, salt, iterations) {
+export async function deriveKey(password, salt, iterations, numBits = DEFAULT_BITSIZE) {
     const subtleCrypto = global.crypto.subtle;
     const TextEncoder = global.TextEncoder;
     if (!subtleCrypto || !TextEncoder) {
@@ -73,7 +76,7 @@ export async function deriveKey(password, salt, iterations) {
             hash: 'SHA-512',
         },
         key,
-        global.Olm.PRIVATE_KEY_LENGTH * 8,
+        numBits,
     );
 
     return new Uint8Array(keybits);

src/crypto/olmlib.js

@@ -113,6 +113,57 @@ export async function encryptMessageForDevice(
     );
 }
 
+/**
+ * Get the existing olm sessions for the given devices, and the devices that
+ * don't have olm sessions.
+ *
+ * @param {module:crypto/OlmDevice} olmDevice
+ *
+ * @param {module:base-apis~MatrixBaseApis} baseApis
+ *
+ * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
+ *    map from userid to list of devices to ensure sessions for
+ *
+ * @return {Promise} resolves to an array.  The first element of the array is a
+ *    a map of user IDs to arrays of deviceInfo, representing the devices that
+ *    don't have established olm sessions.  The second element of the array is
+ *    a map from userId to deviceId to {@link module:crypto~OlmSessionResult}
+ */
+export async function getExistingOlmSessions(
+    olmDevice, baseApis, devicesByUser,
+) {
+    const devicesWithoutSession = {};
+    const sessions = {};
+
+    const promises = [];
+
+    for (const [userId, devices] of Object.entries(devicesByUser)) {
+        for (const deviceInfo of devices) {
+            const deviceId = deviceInfo.deviceId;
+            const key = deviceInfo.getIdentityKey();
+            promises.push((async () => {
+                const sessionId = await olmDevice.getSessionIdForDevice(
+                    key, true,
+                );
+                if (sessionId === null) {
+                    devicesWithoutSession[userId] = devicesWithoutSession[userId] || [];
+                    devicesWithoutSession[userId].push(deviceInfo);
+                } else {
+                    sessions[userId] = sessions[userId] || {};
+                    sessions[userId][deviceId] = {
+                        device: deviceInfo,
+                        sessionId: sessionId,
+                    };
+                }
+            })());
+        }
+    }
+
+    await Promise.all(promises);
+
+    return [devicesWithoutSession, sessions];
+}
+
 /**
  * Try to make sure we have established olm sessions for the given devices.
  *
@@ -123,30 +174,37 @@ export async function encryptMessageForDevice(
  * @param {object<string, module:crypto/deviceinfo[]>} devicesByUser
  *    map from userid to list of devices to ensure sessions for
  *
- * @param {boolean} force If true, establish a new session even if one already exists.
- *     Optional.
+ * @param {boolean} [force=false] If true, establish a new session even if one
+ *     already exists.
+ *
+ * @param {Number} [otkTimeout] The timeout in milliseconds when requesting
+ *     one-time keys for establishing new olm sessions.
+ *
+ * @param {Array} [failedServers] An array to fill with remote servers that
+ *     failed to respond to one-time-key requests.
  *
  * @return {Promise} resolves once the sessions are complete, to
  *    an Object mapping from userId to deviceId to
  *    {@link module:crypto~OlmSessionResult}
  */
 export async function ensureOlmSessionsForDevices(
-    olmDevice, baseApis, devicesByUser, force,
+    olmDevice, baseApis, devicesByUser, force, otkTimeout, failedServers,
 ) {
+    if (typeof force === "number") {
+        failedServers = otkTimeout;
+        otkTimeout = force;
+        force = false;
+    }
+
     const devicesWithoutSession = [
         // [userId, deviceId], ...
     ];
     const result = {};
     const resolveSession = {};
 
-    for (const userId in devicesByUser) {
-        if (!devicesByUser.hasOwnProperty(userId)) {
-            continue;
-        }
+    for (const [userId, devices] of Object.entries(devicesByUser)) {
         result[userId] = {};
-        const devices = devicesByUser[userId];
-        for (let j = 0; j < devices.length; j++) {
-            const deviceInfo = devices[j];
+        for (const deviceInfo of devices) {
             const deviceId = deviceInfo.deviceId;
             const key = deviceInfo.getIdentityKey();
             if (!olmDevice._sessionsInProgress[key]) {
@@ -197,7 +255,7 @@ export async function ensureOlmSessionsForDevices(
     let res;
     try {
         res = await baseApis.claimOneTimeKeys(
-            devicesWithoutSession, oneTimeKeyAlgorithm,
+            devicesWithoutSession, oneTimeKeyAlgorithm, otkTimeout,
         );
     } catch (e) {
         for (const resolver of Object.values(resolveSession)) {
@@ -207,14 +265,14 @@ export async function ensureOlmSessionsForDevices(
         throw e;
     }
 
+    if (failedServers && "failures" in res) {
+        failedServers.push(...Object.keys(res.failures));
+    }
+
     const otk_res = res.one_time_keys || {};
     const promises = [];
-    for (const userId in devicesByUser) {
-        if (!devicesByUser.hasOwnProperty(userId)) {
-            continue;
-        }
+    for (const [userId, devices] of Object.entries(devicesByUser)) {
         const userRes = otk_res[userId] || {};
-        const devices = devicesByUser[userId];
         for (let j = 0; j < devices.length; j++) {
             const deviceInfo = devices[j];
             const deviceId = deviceInfo.deviceId;
@@ -407,6 +465,15 @@ export function encodeBase64(uint8Array) {
     return Buffer.from(uint8Array).toString("base64");
 }
 
+/**
+ * Encode a typed array of uint8 as unpadded base64.
+ * @param {Uint8Array} uint8Array The data to encode.
+ * @return {string} The unpadded base64.
+ */
+export function encodeUnpaddedBase64(uint8Array) {
+    return encodeBase64(uint8Array).replace(/=+$/g, '');
+}
+
 /**
  * Decode a base64 string to a typed array of uint8.
  * @param {string} base64 The base64 to decode.

src/crypto/store/indexeddb-crypto-store-backend.js

@@ -341,18 +341,39 @@ export class Backend {
         };
     }
 
+    getSecretStorePrivateKey(txn, func, type) {
+        const objectStore = txn.objectStore("account");
+        const getReq = objectStore.get(`ssss_cache:${type}`);
+        getReq.onsuccess = function() {
+            try {
+                func(getReq.result || null);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
+        };
+    }
+
     storeCrossSigningKeys(txn, keys) {
         const objectStore = txn.objectStore("account");
         objectStore.put(keys, "crossSigningKeys");
     }
 
+    storeSecretStorePrivateKey(txn, type, key) {
+        const objectStore = txn.objectStore("account");
+        objectStore.put(key, `ssss_cache:${type}`);
+    }
+
     // Olm Sessions
 
     countEndToEndSessions(txn, func) {
         const objectStore = txn.objectStore("sessions");
         const countReq = objectStore.count();
         countReq.onsuccess = function() {
-            func(countReq.result);
+            try {
+                func(countReq.result);
+            } catch (e) {
+                abortWithException(txn, e);
+            }
         };
     }
 
@@ -402,16 +423,16 @@ export class Backend {
         const objectStore = txn.objectStore("sessions");
         const getReq = objectStore.openCursor();
         getReq.onsuccess = function() {
-            const cursor = getReq.result;
-            if (cursor) {
-                func(cursor.value);
-                cursor.continue();
-            } else {
-                try {
+            try {
+                const cursor = getReq.result;
+                if (cursor) {
+                    func(cursor.value);
+                    cursor.continue();
+                } else {
                     func(null);
-                } catch (e) {
-                    abortWithException(txn, e);
                 }
+            } catch (e) {
+                abortWithException(txn, e);
             }
         };
     }

src/crypto/store/indexeddb-crypto-store.js

@@ -46,6 +46,7 @@ export class IndexedDBCryptoStore {
         this._indexedDB = indexedDB;
         this._dbName = dbName;
         this._backendPromise = null;
+        this._backend = null;
     }
 
     static exists(indexedDB, dbName) {
@@ -56,10 +57,12 @@ export class IndexedDBCryptoStore {
      * Ensure the database exists and is up-to-date, or fall back to
      * a local storage or in-memory store.
      *
+     * This must be called before the store can be used.
+     *
      * @return {Promise} resolves to either an IndexedDBCryptoStoreBackend.Backend,
      * or a MemoryCryptoStore
      */
-    _connect() {
+    startup() {
         if (this._backendPromise) {
             return this._backendPromise;
         }
@@ -135,6 +138,8 @@ export class IndexedDBCryptoStore {
                 );
                 return new MemoryCryptoStore();
             }
+        }).then(backend => {
+            this._backend = backend;
         });
 
         return this._backendPromise;
@@ -189,9 +194,7 @@ export class IndexedDBCryptoStore {
      *    same instance as passed in, or the existing one.
      */
     getOrAddOutgoingRoomKeyRequest(request) {
-        return this._connect().then((backend) => {
-            return backend.getOrAddOutgoingRoomKeyRequest(request);
-        });
+        return this._backend.getOrAddOutgoingRoomKeyRequest(request);
     }
 
     /**
@@ -205,9 +208,7 @@ export class IndexedDBCryptoStore {
      *    not found
      */
     getOutgoingRoomKeyRequest(requestBody) {
-        return this._connect().then((backend) => {
-            return backend.getOutgoingRoomKeyRequest(requestBody);
-        });
+        return this._backend.getOutgoingRoomKeyRequest(requestBody);
     }
 
     /**
@@ -221,9 +222,7 @@ export class IndexedDBCryptoStore {
      *    requests in those states, an arbitrary one is chosen.
      */
     getOutgoingRoomKeyRequestByState(wantedStates) {
-        return this._connect().then((backend) => {
-            return backend.getOutgoingRoomKeyRequestByState(wantedStates);
-        });
+        return this._backend.getOutgoingRoomKeyRequestByState(wantedStates);
     }
 
     /**
@@ -237,11 +236,9 @@ export class IndexedDBCryptoStore {
      *    {@link module:crypto/store/base~OutgoingRoomKeyRequest}
      */
     getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
-        return this._connect().then((backend) => {
-            return backend.getOutgoingRoomKeyRequestsByTarget(
-                userId, deviceId, wantedStates,
-            );
-        });
+        return this._backend.getOutgoingRoomKeyRequestsByTarget(
+            userId, deviceId, wantedStates,
+        );
     }
 
     /**
@@ -257,11 +254,9 @@ export class IndexedDBCryptoStore {
      *    updated request, or null if no matching row was found
      */
     updateOutgoingRoomKeyRequest(requestId, expectedState, updates) {
-        return this._connect().then((backend) => {
-            return backend.updateOutgoingRoomKeyRequest(
-                requestId, expectedState, updates,
-            );
-        });
+        return this._backend.updateOutgoingRoomKeyRequest(
+            requestId, expectedState, updates,
+        );
     }
 
     /**
@@ -274,9 +269,7 @@ export class IndexedDBCryptoStore {
      * @returns {Promise} resolves once the operation is completed
      */
     deleteOutgoingRoomKeyRequest(requestId, expectedState) {
-        return this._connect().then((backend) => {
-            return backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
-        });
+        return this._backend.deleteOutgoingRoomKeyRequest(requestId, expectedState);
     }
 
     // Olm Account
@@ -289,9 +282,7 @@ export class IndexedDBCryptoStore {
      * @param {function(string)} func Called with the account pickle
      */
     getAccount(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getAccount(txn, func);
-        });
+        this._backend.getAccount(txn, func);
     }
 
     /**
@@ -302,9 +293,7 @@ export class IndexedDBCryptoStore {
      * @param {string} newData The new account pickle to store.
      */
     storeAccount(txn, newData) {
-        this._backendPromise.then(backend => {
-            backend.storeAccount(txn, newData);
-        });
+        this._backend.storeAccount(txn, newData);
     }
 
     /**
@@ -316,9 +305,16 @@ export class IndexedDBCryptoStore {
      *        { key_type: base64 encoded seed } where key type = user_signing_key_seed or self_signing_key_seed
      */
     getCrossSigningKeys(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getCrossSigningKeys(txn, func);
-        });
+        this._backend.getCrossSigningKeys(txn, func);
+    }
+
+    /**
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {function(string)} func Called with the private key
+     * @param {string} type A key type
+     */
+    getSecretStorePrivateKey(txn, func, type) {
+        this._backend.getSecretStorePrivateKey(txn, func, type);
     }
 
     /**
@@ -328,9 +324,18 @@ export class IndexedDBCryptoStore {
      * @param {string} keys keys object as getCrossSigningKeys()
      */
     storeCrossSigningKeys(txn, keys) {
-        this._backendPromise.then(backend => {
-            backend.storeCrossSigningKeys(txn, keys);
-        });
+        this._backend.storeCrossSigningKeys(txn, keys);
+    }
+
+    /**
+     * Write the cross-signing private keys back to the store
+     *
+     * @param {*} txn An active transaction. See doTxn().
+     * @param {string} type The type of cross-signing private key to store
+     * @param {string} key keys object as getCrossSigningKeys()
+     */
+    storeSecretStorePrivateKey(txn, type, key) {
+        this._backend.storeSecretStorePrivateKey(txn, type, key);
     }
 
     // Olm sessions
@@ -341,9 +346,7 @@ export class IndexedDBCryptoStore {
      * @param {function(int)} func Called with the count of sessions
      */
     countEndToEndSessions(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.countEndToEndSessions(txn, func);
-        });
+        this._backend.countEndToEndSessions(txn, func);
     }
 
     /**
@@ -359,9 +362,7 @@ export class IndexedDBCryptoStore {
      *     a message.
      */
     getEndToEndSession(deviceKey, sessionId, txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndSession(deviceKey, sessionId, txn, func);
-        });
+        this._backend.getEndToEndSession(deviceKey, sessionId, txn, func);
     }
 
     /**
@@ -376,9 +377,7 @@ export class IndexedDBCryptoStore {
      *     a message.
      */
     getEndToEndSessions(deviceKey, txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndSessions(deviceKey, txn, func);
-        });
+        this._backend.getEndToEndSessions(deviceKey, txn, func);
     }
 
     /**
@@ -389,9 +388,7 @@ export class IndexedDBCryptoStore {
      *     and session keys.
      */
     getAllEndToEndSessions(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getAllEndToEndSessions(txn, func);
-        });
+        this._backend.getAllEndToEndSessions(txn, func);
     }
 
     /**
@@ -402,29 +399,21 @@ export class IndexedDBCryptoStore {
      * @param {*} txn An active transaction. See doTxn().
      */
     storeEndToEndSession(deviceKey, sessionId, sessionInfo, txn) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndSession(
-                deviceKey, sessionId, sessionInfo, txn,
-            );
-        });
+        this._backend.storeEndToEndSession(
+            deviceKey, sessionId, sessionInfo, txn,
+        );
     }
 
     storeEndToEndSessionProblem(deviceKey, type, fixed) {
-        return this._backendPromise.then(async (backend) => {
-            await backend.storeEndToEndSessionProblem(deviceKey, type, fixed);
-        });
+        return this._backend.storeEndToEndSessionProblem(deviceKey, type, fixed);
     }
 
     getEndToEndSessionProblem(deviceKey, timestamp) {
-        return this._backendPromise.then(async (backend) => {
-            return await backend.getEndToEndSessionProblem(deviceKey, timestamp);
-        });
+        return this._backend.getEndToEndSessionProblem(deviceKey, timestamp);
     }
 
     filterOutNotifiedErrorDevices(devices) {
-        return this._backendPromise.then(async (backend) => {
-            return await backend.filterOutNotifiedErrorDevices(devices);
-        });
+        return this._backend.filterOutNotifiedErrorDevices(devices);
     }
 
     // Inbound group sessions
@@ -439,11 +428,9 @@ export class IndexedDBCryptoStore {
      *     to Base64 end-to-end session.
      */
     getEndToEndInboundGroupSession(senderCurve25519Key, sessionId, txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndInboundGroupSession(
-                senderCurve25519Key, sessionId, txn, func,
-            );
-        });
+        this._backend.getEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, txn, func,
+        );
     }
 
     /**
@@ -454,9 +441,7 @@ export class IndexedDBCryptoStore {
      *     sessionData}, then once with null to indicate the end of the list.
      */
     getAllEndToEndInboundGroupSessions(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getAllEndToEndInboundGroupSessions(txn, func);
-        });
+        this._backend.getAllEndToEndInboundGroupSessions(txn, func);
     }
 
     /**
@@ -469,11 +454,9 @@ export class IndexedDBCryptoStore {
      * @param {*} txn An active transaction. See doTxn().
      */
     addEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
-        this._backendPromise.then(backend => {
-            backend.addEndToEndInboundGroupSession(
-                senderCurve25519Key, sessionId, sessionData, txn,
-            );
-        });
+        this._backend.addEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
     }
 
     /**
@@ -486,21 +469,17 @@ export class IndexedDBCryptoStore {
      * @param {*} txn An active transaction. See doTxn().
      */
     storeEndToEndInboundGroupSession(senderCurve25519Key, sessionId, sessionData, txn) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndInboundGroupSession(
-                senderCurve25519Key, sessionId, sessionData, txn,
-            );
-        });
+        this._backend.storeEndToEndInboundGroupSession(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
     }
 
     storeEndToEndInboundGroupSessionWithheld(
         senderCurve25519Key, sessionId, sessionData, txn,
     ) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndInboundGroupSessionWithheld(
-                senderCurve25519Key, sessionId, sessionData, txn,
-            );
-        });
+        this._backend.storeEndToEndInboundGroupSessionWithheld(
+            senderCurve25519Key, sessionId, sessionData, txn,
+        );
     }
 
     // End-to-end device tracking
@@ -516,9 +495,7 @@ export class IndexedDBCryptoStore {
      * @param {*} txn An active transaction. See doTxn().
      */
     storeEndToEndDeviceData(deviceData, txn) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndDeviceData(deviceData, txn);
-        });
+        this._backend.storeEndToEndDeviceData(deviceData, txn);
     }
 
     /**
@@ -529,9 +506,7 @@ export class IndexedDBCryptoStore {
      *     device data
      */
     getEndToEndDeviceData(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndDeviceData(txn, func);
-        });
+        this._backend.getEndToEndDeviceData(txn, func);
     }
 
     // End to End Rooms
@@ -543,9 +518,7 @@ export class IndexedDBCryptoStore {
      * @param {*} txn An active transaction. See doTxn().
      */
     storeEndToEndRoom(roomId, roomInfo, txn) {
-        this._backendPromise.then(backend => {
-            backend.storeEndToEndRoom(roomId, roomInfo, txn);
-        });
+        this._backend.storeEndToEndRoom(roomId, roomInfo, txn);
     }
 
     /**
@@ -554,9 +527,7 @@ export class IndexedDBCryptoStore {
      * @param {function(Object)} func Function called with the end to end encrypted rooms
      */
     getEndToEndRooms(txn, func) {
-        this._backendPromise.then(backend => {
-            backend.getEndToEndRooms(txn, func);
-        });
+        this._backend.getEndToEndRooms(txn, func);
     }
 
     // session backups
@@ -568,9 +539,7 @@ export class IndexedDBCryptoStore {
      * @returns {Promise} resolves to an array of inbound group sessions
      */
     getSessionsNeedingBackup(limit) {
-        return this._connect().then((backend) => {
-            return backend.getSessionsNeedingBackup(limit);
-        });
+        return this._backend.getSessionsNeedingBackup(limit);
     }
 
     /**
@@ -579,9 +548,7 @@ export class IndexedDBCryptoStore {
      * @returns {Promise} resolves to the number of sessions
      */
     countSessionsNeedingBackup(txn) {
-        return this._connect().then((backend) => {
-            return backend.countSessionsNeedingBackup(txn);
-        });
+        return this._backend.countSessionsNeedingBackup(txn);
     }
 
     /**
@@ -591,9 +558,7 @@ export class IndexedDBCryptoStore {
      * @returns {Promise} resolves when the sessions are unmarked
      */
     unmarkSessionsNeedingBackup(sessions, txn) {
-        return this._connect().then((backend) => {
-            return backend.unmarkSessionsNeedingBackup(sessions, txn);
-        });
+        return this._backend.unmarkSessionsNeedingBackup(sessions, txn);
     }
 
     /**
@@ -603,9 +568,7 @@ export class IndexedDBCryptoStore {
      * @returns {Promise} resolves when the sessions are marked
      */
     markSessionsNeedingBackup(sessions, txn) {
-        return this._connect().then((backend) => {
-            return backend.markSessionsNeedingBackup(sessions, txn);
-        });
+        return this._backend.markSessionsNeedingBackup(sessions, txn);
     }
 
     /**
@@ -630,9 +593,7 @@ export class IndexedDBCryptoStore {
      *     exception will propagate to the caller of the getFoo method.
      */
     doTxn(mode, stores, func) {
-        return this._connect().then((backend) => {
-            return backend.doTxn(mode, stores, func);
-        });
+        return this._backend.doTxn(mode, stores, func);
     }
 }
 

src/crypto/store/localStorage-crypto-store.js

@@ -367,12 +367,23 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {
         func(keys);
     }
 
+    getSecretStorePrivateKey(txn, func, type) {
+        const key = getJsonItem(this.store, E2E_PREFIX + `ssss_cache.${type}`);
+        func(key ? Uint8Array.from(key) : key);
+    }
+
     storeCrossSigningKeys(txn, keys) {
         setJsonItem(
             this.store, KEY_CROSS_SIGNING_KEYS, keys,
         );
     }
 
+    storeSecretStorePrivateKey(txn, type, key) {
+        setJsonItem(
+            this.store, E2E_PREFIX + `ssss_cache.${type}`, Array.from(key),
+        );
+    }
+
     doTxn(mode, stores, func) {
         return Promise.resolve(func(null));
     }

src/crypto/store/memory-crypto-store.js

@@ -33,6 +33,8 @@ export class MemoryCryptoStore {
         this._outgoingRoomKeyRequests = [];
         this._account = null;
         this._crossSigningKeys = null;
+        this._privateKeys = {};
+        this._backupKeys = {};
 
         // Map of {devicekey -> {sessionId -> session pickle}}
         this._sessions = {};
@@ -51,6 +53,18 @@ export class MemoryCryptoStore {
         this._sessionsNeedingBackup = {};
     }
 
+    /**
+     * Ensure the database exists and is up-to-date.
+     *
+     * This must be called before the store can be used.
+     *
+     * @return {Promise} resolves to the store.
+     */
+    async startup() {
+        // No startup work to do for the memory store.
+        return this;
+    }
+
     /**
      * Delete all data from this store.
      *
@@ -243,10 +257,19 @@ export class MemoryCryptoStore {
         func(this._crossSigningKeys);
     }
 
+    getSecretStorePrivateKey(txn, func, type) {
+        const result = this._privateKeys[type];
+        return func(result || null);
+    }
+
     storeCrossSigningKeys(txn, keys) {
         this._crossSigningKeys = keys;
     }
 
+    storeSecretStorePrivateKey(txn, type, key) {
+        this._privateKeys[type] = key;
+    }
+
     // Olm Sessions
 
     countEndToEndSessions(txn, func) {

src/crypto/verification/Base.js

@@ -24,6 +24,8 @@ import {EventEmitter} from 'events';
 import {logger} from '../../logger';
 import {DeviceInfo} from '../deviceinfo';
 import {newTimeoutError} from "./Error";
+import {CrossSigningInfo} from "../CrossSigning";
+import {decodeBase64} from "../olmlib";
 
 const timeoutException = new Error("Verification timed out");
 
@@ -76,6 +78,8 @@ export class VerificationBase extends EventEmitter {
         this._transactionTimeoutTimer = null;
     }
 
+    static keyRequestTimeoutMs = 1000 * 60;
+
     get initiatedByMe() {
         // if there is no start event yet,
         // we probably want to send it,
@@ -130,6 +134,8 @@ export class VerificationBase extends EventEmitter {
 
     switchStartEvent(event) {
         if (this.canSwitchStartEvent(event)) {
+            logger.log("Verification Base: switching verification start event",
+                {restartingFlow: !!this._rejectEvent});
             if (this._rejectEvent) {
                 const reject = this._rejectEvent;
                 this._rejectEvent = undefined;
@@ -155,10 +161,13 @@ export class VerificationBase extends EventEmitter {
         } else if (e.getType() === "m.key.verification.cancel") {
             const reject = this._reject;
             this._reject = undefined;
-            const content = e.getContent();
-            const {reason, code} = content;
-            reject(new Error(`Other side cancelled verification ` +
-                `because ${reason} (${code})`));
+            // there is only promise to reject if verify has been called
+            if (reject) {
+                const content = e.getContent();
+                const {reason, code} = content;
+                reject(new Error(`Other side cancelled verification ` +
+                    `because ${reason} (${code})`));
+            }
         } else if (this._expectedEvent) {
             // only cancel if there is an event expected.
             // if there is no event expected, it means verify() wasn't called
@@ -181,11 +190,65 @@ export class VerificationBase extends EventEmitter {
     done() {
         this._endTimer(); // always kill the activity timer
         if (!this._done) {
-            if (this._channel.needsDoneMessage) {
-                // verification in DM requires a done message
-                this._send("m.key.verification.done", {});
-            }
+            this.request.onVerifierFinished();
             this._resolve();
+
+            //#region Cross-signing keys request
+            // If this is a self-verification, ask the other party for keys
+            if (this._baseApis.getUserId() !== this.userId) {
+                return;
+            }
+            console.log("VerificationBase.done: Self-verification done; requesting keys");
+            /* This happens asynchronously, and we're not concerned about
+             * waiting for it.  We return here in order to test. */
+            return new Promise((resolve, reject) => {
+                const client = this._baseApis;
+                const original = client._crypto._crossSigningInfo;
+                const storage = client._crypto._secretStorage;
+
+                /* We already have all of the infrastructure we need to validate and
+                 * cache cross-signing keys, so instead of replicating that, here we
+                 * set up callbacks that request them from the other device and call
+                 * CrossSigningInfo.getCrossSigningKey() to validate/cache */
+                const crossSigning = new CrossSigningInfo(
+                    original.userId,
+                    { getCrossSigningKey: async (type) => {
+                        console.debug("VerificationBase.done: requesting secret",
+                                      type, this.deviceId);
+                        const { promise } =
+                            storage.request(`m.cross_signing.${type}`, [this.deviceId]);
+                        const result = await promise;
+                        const decoded = decodeBase64(result);
+                        return Uint8Array.from(decoded);
+                    } },
+                    original._cacheCallbacks,
+                );
+                crossSigning.keys = original.keys;
+
+                // XXX: get all keys out if we get one key out
+                // https://github.com/vector-im/riot-web/issues/12604
+                // then change here to reject on the timeout
+                /* Requests can be ignored, so don't wait around forever */
+                const timeout = new Promise((resolve, reject) => {
+                    setTimeout(
+                        resolve,
+                        VerificationBase.keyRequestTimeoutMs,
+                        new Error("Timeout"),
+                    );
+                });
+
+                /* We call getCrossSigningKey() for its side-effects */
+                return Promise.race([
+                    Promise.all([
+                        crossSigning.getCrossSigningKey("self_signing"),
+                        crossSigning.getCrossSigningKey("user_signing"),
+                    ]),
+                    timeout,
+                ]).then(resolve, reject);
+            }).catch((e) => {
+                console.warn("VerificationBase: failure while requesting keys:", e);
+            });
+            //#endregion
         }
     }
 

src/crypto/verification/QRCode.js

@@ -80,7 +80,7 @@ export class ReciprocateQRCode extends Base {
 
         const devices = (await this._baseApis.getStoredDevicesForUser(this.userId)) || [];
         const targetDevice = devices.find(d => {
-            return d.deviceId === this.request.estimatedTargetDevice.deviceId;
+            return d.deviceId === this.request.targetDevice.deviceId;
         });
         if (!targetDevice) throw new Error("Device not found, somehow");
         keys[`ed25519:${targetDevice.deviceId}`] = targetDevice.getFingerprint();

src/crypto/verification/SAS.js

@@ -28,6 +28,7 @@ import {
     newUnknownMethodError,
     newUserCancelledError,
 } from './Error';
+import {logger} from '../../logger';
 
 const START_TYPE = "m.key.verification.start";
 
@@ -165,6 +166,15 @@ const macMethods = {
     "hmac-sha256": "calculate_mac_long_kdf",
 };
 
+function calculateMAC(olmSAS, method) {
+    return function(...args) {
+        const macFunction = olmSAS[macMethods[method]];
+        const mac = macFunction.apply(olmSAS, args);
+        logger.log("SAS calculateMAC:", method, args, mac);
+        return mac;
+    };
+}
+
 /* lists of algorithms/methods that are supported.  The key agreement, hashes,
  * and MAC lists should be sorted in order of preference (most preferred
  * first).
@@ -306,7 +316,7 @@ export class SAS extends Base {
                   + this._channel.transactionId;
             const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
             const verifySAS = new Promise((resolve, reject) => {
-                this.emit("show_sas", {
+                this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
                     confirm: () => {
                         this._sendMAC(olmSAS, macMethod);
@@ -314,7 +324,8 @@ export class SAS extends Base {
                     },
                     cancel: () => reject(newUserCancelledError()),
                     mismatch: () => reject(newMismatchedSASError()),
-                });
+                };
+                this.emit("show_sas", this.sasEvent);
             });
 
 
@@ -390,7 +401,7 @@ export class SAS extends Base {
                   + this._channel.transactionId;
             const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
             const verifySAS = new Promise((resolve, reject) => {
-                this.emit("show_sas", {
+                this.sasEvent = {
                     sas: generateSas(sasBytes, sasMethods),
                     confirm: () => {
                         this._sendMAC(olmSAS, macMethod);
@@ -398,7 +409,8 @@ export class SAS extends Base {
                     },
                     cancel: () => reject(newUserCancelledError()),
                     mismatch: () => reject(newMismatchedSASError()),
-                });
+                };
+                this.emit("show_sas", this.sasEvent);
             });
 
 
@@ -429,7 +441,7 @@ export class SAS extends Base {
               + this._channel.transactionId;
 
         const deviceKeyId = `ed25519:${this._baseApis.deviceId}`;
-        mac[deviceKeyId] = olmSAS[macMethods[method]](
+        mac[deviceKeyId] = calculateMAC(olmSAS, method)(
             this._baseApis.getDeviceEd25519Key(),
             baseInfo + deviceKeyId,
         );
@@ -438,14 +450,14 @@ export class SAS extends Base {
         const crossSigningId = this._baseApis.getCrossSigningId();
         if (crossSigningId) {
             const crossSigningKeyId = `ed25519:${crossSigningId}`;
-            mac[crossSigningKeyId] = olmSAS[macMethods[method]](
+            mac[crossSigningKeyId] = calculateMAC(olmSAS, method)(
                 crossSigningId,
                 baseInfo + crossSigningKeyId,
             );
             keyList.push(crossSigningKeyId);
         }
 
-        const keys = olmSAS[macMethods[method]](
+        const keys = calculateMAC(olmSAS, method)(
             keyList.sort().join(","),
             baseInfo + "KEY_IDS",
         );
@@ -458,7 +470,7 @@ export class SAS extends Base {
               + this._baseApis.getUserId() + this._baseApis.deviceId
               + this._channel.transactionId;
 
-        if (content.keys !== olmSAS[macMethods[method]](
+        if (content.keys !== calculateMAC(olmSAS, method)(
             Object.keys(content.mac).sort().join(","),
             baseInfo + "KEY_IDS",
         )) {
@@ -466,7 +478,7 @@ export class SAS extends Base {
         }
 
         await this._verifyKeys(this.userId, content.mac, (keyId, device, keyInfo) => {
-            if (keyInfo !== olmSAS[macMethods[method]](
+            if (keyInfo !== calculateMAC(olmSAS, method)(
                 device.keys[keyId],
                 baseInfo + keyId,
             )) {

src/crypto/verification/request/InRoomChannel.js

@@ -305,7 +305,6 @@ export class InRoomRequests {
     getRequest(event) {
         const roomId = event.getRoomId();
         const txnId = InRoomChannel.getTransactionId(event);
-//        console.log(`looking for request in room ${roomId} with txnId ${txnId} for an ${event.getType()} from ${event.getSender()}...`);
         return this._getRequestByTxnId(roomId, txnId);
     }
 
@@ -351,4 +350,15 @@ export class InRoomRequests {
             }
         }
     }
+
+    findRequestInProgress(roomId) {
+        const requestsByTxnId = this._requestsByRoomId.get(roomId);
+        if (requestsByTxnId) {
+            for (const request of requestsByTxnId.values()) {
+                if (request.pending) {
+                    return request;
+                }
+            }
+        }
+    }
 }

src/crypto/verification/request/ToDeviceChannel.js

@@ -43,12 +43,26 @@ export class ToDeviceChannel {
         this._deviceId = deviceId;
     }
 
+    isToDevices(devices) {
+        if (devices.length === this._devices.length) {
+            for (const device of devices) {
+                const d = this._devices.find(d => d.deviceId === device.deviceId);
+                if (!d) {
+                    return false;
+                }
+            }
+            return true;
+        } else {
+            return false;
+        }
+    }
+
     get deviceId() {
         return this._deviceId;
     }
 
     get needsDoneMessage() {
-        return true;
+        return false;
     }
 
     static getEventType(event) {
@@ -335,4 +349,15 @@ export class ToDeviceRequests {
             }
         }
     }
+
+    findRequestInProgress(userId, devices) {
+        const requestsByTxnId = this._requestsByUserId.get(userId);
+        if (requestsByTxnId) {
+            for (const request of requestsByTxnId.values()) {
+                if (request.pending && request.channel.isToDevices(devices)) {
+                    return request;
+                }
+            }
+        }
+    }
 }

src/crypto/verification/request/VerificationRequest.js

@@ -71,6 +71,9 @@ export class VerificationRequest extends EventEmitter {
         this._observeOnly = false;
         this._timeoutTimer = null;
         this._sharedSecret = null; // used for QR codes
+        this._accepting = false;
+        this._declining = false;
+        this._verifierHasFinished = false;
     }
 
     /**
@@ -179,10 +182,59 @@ export class VerificationRequest extends EventEmitter {
         return this._verifier;
     }
 
+    get canAccept() {
+        return this.phase < PHASE_READY && !this._accepting && !this._declining;
+    }
+
+    get accepting() {
+        return this._accepting;
+    }
+
+    get declining() {
+        return this._declining;
+    }
+
     /** whether this request has sent it's initial event and needs more events to complete */
     get pending() {
-        return this._phase !== PHASE_DONE
-            && this._phase !== PHASE_CANCELLED;
+        return !this.observeOnly &&
+            this._phase !== PHASE_DONE &&
+            this._phase !== PHASE_CANCELLED;
+    }
+
+    /** Checks whether the other party supports a given verification method.
+     *  This is useful when setting up the QR code UI, as it is somewhat asymmetrical:
+     *  if the other party supports SCAN_QR, we should show a QR code in the UI, and vice versa.
+     *  For methods that need to be supported by both ends, use the `methods` property.
+     *  @param {string} method the method to check
+     *  @return {bool} whether or not the other party said the supported the method */
+    otherPartySupportsMethod(method) {
+        if (!this.ready && !this.started) {
+            return false;
+        }
+        const theirMethodEvent = this._eventsByThem.get(REQUEST_TYPE) ||
+            this._eventsByThem.get(READY_TYPE);
+        if (!theirMethodEvent) {
+            // if we started straight away with .start event,
+            // we are assuming that the other side will support the
+            // chosen method, so return true for that.
+            if (this.started && this.initiatedByMe) {
+                const myStartEvent = this._eventsByUs.get(START_TYPE);
+                const content = myStartEvent && myStartEvent.getContent();
+                const myStartMethod = content && content.method;
+                return method == myStartMethod;
+            }
+            return false;
+        }
+        const content = theirMethodEvent.getContent();
+        if (!content) {
+            return false;
+        }
+        const {methods} = content;
+        if (!Array.isArray(methods)) {
+            return false;
+        }
+
+        return methods.includes(method);
     }
 
     /** Whether this request was initiated by the syncing user.
@@ -342,6 +394,8 @@ export class VerificationRequest extends EventEmitter {
      */
     async cancel({reason = "User declined", code = "m.user"} = {}) {
         if (!this.observeOnly && this._phase !== PHASE_CANCELLED) {
+            this._declining = true;
+            this.emit("change");
             if (this._verifier) {
                 return this._verifier.cancel(errorFactory(code, reason)());
             } else {
@@ -358,15 +412,17 @@ export class VerificationRequest extends EventEmitter {
     async accept() {
         if (!this.observeOnly && this.phase === PHASE_REQUESTED && !this.initiatedByMe) {
             const methods = [...this._verificationMethods.keys()];
+            this._accepting = true;
+            this.emit("change");
             await this.channel.send(READY_TYPE, {methods});
             this._generateSharedSecret();
         }
     }
 
     _generateSharedSecret() {
-        const secretBytes = new Uint8Array(32); // 256bits
+        const secretBytes = new Uint8Array(8);
         global.crypto.getRandomValues(secretBytes);
-        this._sharedSecret = olmlib.encodeBase64(secretBytes);
+        this._sharedSecret = olmlib.encodeUnpaddedBase64(secretBytes);
     }
 
     /**
@@ -459,7 +515,7 @@ export class VerificationRequest extends EventEmitter {
         }
 
         const ourDoneEvent = this._eventsByUs.get(DONE_TYPE);
-        if (ourDoneEvent && phase() === PHASE_STARTED) {
+        if (this._verifierHasFinished || (ourDoneEvent && phase() === PHASE_STARTED)) {
             transitions.push({phase: PHASE_DONE});
         }
 
@@ -507,6 +563,55 @@ export class VerificationRequest extends EventEmitter {
         }
     }
 
+    _applyPhaseTransitions() {
+        const transitions = this._calculatePhaseTransitions();
+        const existingIdx = transitions.findIndex(t => t.phase === this.phase);
+        // trim off phases we already went through, if any
+        const newTransitions = transitions.slice(existingIdx + 1);
+        // transition to all new phases
+        for (const transition of newTransitions) {
+            this._transitionToPhase(transition);
+        }
+        return newTransitions;
+    }
+
+    _isWinningStartRace(newEvent) {
+        if (newEvent.getType() !== START_TYPE) {
+            return false;
+        }
+        const oldEvent = this._verifier.startEvent;
+        const isSelfVerification = this.channel.userId === this._client.getUserId();
+
+        let oldRaceIdentifier;
+        if (isSelfVerification) {
+            // if the verifier does not have a startEvent,
+            // it is because it's still sending and we are on the initator side
+            // we know we are sending a .start event because we already
+            // have a verifier (checked in calling method)
+            if (oldEvent) {
+                const oldContent = oldEvent.getContent();
+                oldRaceIdentifier = oldContent && oldContent.from_device;
+            } else {
+                oldRaceIdentifier = this._client.getDeviceId();
+            }
+        } else {
+            if (oldEvent) {
+                oldRaceIdentifier = oldEvent.getSender();
+            } else {
+                oldRaceIdentifier = this._client.getUserId();
+            }
+        }
+
+        let newRaceIdentifier;
+        if (isSelfVerification) {
+            const newContent = newEvent.getContent();
+            newRaceIdentifier = newContent && newContent.from_device;
+        } else {
+            newRaceIdentifier = newEvent.getSender();
+        }
+        return newRaceIdentifier < oldRaceIdentifier;
+    }
+
     /**
      * Changes the state of the request and verifier in response to a key verification event.
      * @param {string} type the "symbolic" event type, as returned by the `getEventType` function on the channel.
@@ -519,7 +624,7 @@ export class VerificationRequest extends EventEmitter {
      */
     async handleEvent(type, event, isLiveEvent, isRemoteEcho, isSentByUs) {
         // if reached phase cancelled or done, ignore anything else that comes
-        if (!this.pending) {
+        if (this.done || this.cancelled) {
             return;
         }
         const wasObserveOnly = this._observeOnly;
@@ -535,24 +640,13 @@ export class VerificationRequest extends EventEmitter {
         const oldPhase = this.phase;
         this._addEvent(type, event, isSentByUs);
 
-        const transitions = this._calculatePhaseTransitions();
-        const existingIdx = transitions.findIndex(t => t.phase === this.phase);
-        // trim off phases we already went through, if any
-        const newTransitions = transitions.slice(existingIdx + 1);
-        // transition to all new phases
-        for (const transition of newTransitions) {
-            this._transitionToPhase(transition);
-        }
+        // this will create if needed the verifier so needs to happen before calling it
+        const newTransitions = this._applyPhaseTransitions();
         try {
             // only pass events from the other side to the verifier,
             // no remote echos of our own events
             if (this._verifier && !this.observeOnly) {
-                const oldEvent = this._verifier.startEvent;
-                // if the verifier does not have a startEvent, it is because it's still sending and we are on the initator side
-                const oldSender = oldEvent ?
-                    oldEvent.getSender() :
-                    this._client.getUserId();
-                const newEventWinsRace = event.getSender() < oldSender;
+                const newEventWinsRace = this._isWinningStartRace(event);
                 if (this._verifier.canSwitchStartEvent(event) && newEventWinsRace) {
                     this._verifier.switchStartEvent(event);
                 } else if (!isRemoteEcho) {
@@ -576,9 +670,11 @@ export class VerificationRequest extends EventEmitter {
         } finally {
             // log events we processed so we can see from rageshakes what events were added to a request
             logger.log(`Verification request ${this.channel.transactionId}: ` +
-                `${type} event with ${JSON.stringify(event.getContent())} ` +
-                `deviceId:${this.channel.deviceId} ` +
-                `sender:${event.getSender()}, isSentByUs:${isSentByUs} ` +
+                `${type} event with id:${event.getId()}, ` +
+                `content:${JSON.stringify(event.getContent())} ` +
+                `deviceId:${this.channel.deviceId}, ` +
+                `sender:${event.getSender()}, isSentByUs:${isSentByUs}, ` +
+                `isLiveEvent:${isLiveEvent}, isRemoteEcho:${isRemoteEcho}, ` +
                 `phase:${oldPhase}=>${this.phase}, ` +
                 `observeOnly:${wasObserveOnly}=>${this._observeOnly}`);
         }
@@ -622,7 +718,12 @@ export class VerificationRequest extends EventEmitter {
 
         const isUnexpectedRequest = type === REQUEST_TYPE && this.phase !== PHASE_UNSENT;
         const isUnexpectedReady = type === READY_TYPE && this.phase !== PHASE_REQUESTED;
-        if (isUnexpectedRequest || isUnexpectedReady) {
+        // only if phase has passed from PHASE_UNSENT should we cancel, because events
+        // are allowed to come in in any order (at least with InRoomChannel). So we only know
+        // we're dealing with a valid request we should participate in once we've moved to PHASE_REQUESTED
+        // before that, we could be looking at somebody elses verification request and we just
+        // happen to be in the room
+        if (this.phase !== PHASE_UNSENT && (isUnexpectedRequest || isUnexpectedReady)) {
             logger.warn(`Cancelling, unexpected ${type} verification ` +
                 `event from ${event.getSender()}`);
             const reason = `Unexpected ${type} event in phase ${this.phase}`;
@@ -704,4 +805,16 @@ export class VerificationRequest extends EventEmitter {
         }
         return true;
     }
+
+    onVerifierFinished() {
+        if (this.channel.needsDoneMessage) {
+            // verification in DM requires a done message
+            this.channel.send("m.key.verification.done", {});
+        }
+        this._verifierHasFinished = true;
+        const newTransitions = this._applyPhaseTransitions();
+        if (newTransitions.length) {
+            this._setPhase(newTransitions[newTransitions.length - 1].phase);
+        }
+    }
 }

src/errors.js

@@ -44,3 +44,10 @@ InvalidCryptoStoreError.prototype = Object.create(Error.prototype, {
   },
 });
 Reflect.setPrototypeOf(InvalidCryptoStoreError, Error);
+
+export class KeySignatureUploadError extends Error {
+  constructor(message, value) {
+    super(message);
+    this.value = value;
+  }
+}

src/http-api.js

@@ -47,6 +47,7 @@ export const PREFIX_UNSTABLE = "/_matrix/client/unstable";
 
 /**
  * URI path for v1 of the the identity API
+ * @deprecated Use v2.
  */
 export const PREFIX_IDENTITY_V1 = "/_matrix/identity/api/v1";
 

src/index.ts

@@ -21,5 +21,12 @@ import request from "request";
 matrixcs.request(request);
 utils.runPolyfills();
 
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    console.log('nodejs was compiled without crypto support');
+}
+
 export * from "./matrix";
 export default matrixcs;

src/logger.js

@@ -29,6 +29,25 @@ import log from "loglevel";
 // Part of #332 is introducing a logging library in the first place.
 const DEFAULT_NAMESPACE = "matrix";
 
+// because rageshakes in react-sdk hijack the console log, also at module load time,
+// initializing the logger here races with the initialization of rageshakes.
+// to avoid the issue, we override the methodFactory of loglevel that binds to the
+// console methods at initialization time by a factory that looks up the console methods
+// when logging so we always get the current value of console methods.
+log.methodFactory = function(methodName, logLevel, loggerName) {
+    return function(...args) {
+        const supportedByConsole = methodName === "error" ||
+            methodName === "warn" ||
+            methodName === "trace" ||
+            methodName === "info";
+        if (supportedByConsole) {
+            return console[methodName](...args);
+        } else {
+            return console.log(...args);
+        }
+    };
+};
+
 /**
  * Drop-in replacement for <code>console</code> using {@link https://www.npmjs.com/package/loglevel|loglevel}.
  * Can be tailored down to specific use cases if needed.

src/models/room.js

@@ -861,11 +861,23 @@ Room.prototype.getAliases = function() {
 Room.prototype.getCanonicalAlias = function() {
     const canonicalAlias = this.currentState.getStateEvents("m.room.canonical_alias", "");
     if (canonicalAlias) {
-        return canonicalAlias.getContent().alias;
+        return canonicalAlias.getContent().alias || null;
     }
     return null;
 };
 
+/**
+ * Get this room's alternative aliases
+ * @return {array} The room's alternative aliases, or an empty array
+ */
+Room.prototype.getAltAliases = function() {
+    const canonicalAlias = this.currentState.getStateEvents("m.room.canonical_alias", "");
+    if (canonicalAlias) {
+        return canonicalAlias.getContent().alt_aliases || [];
+    }
+    return [];
+};
+
 /**
  * Add events to a timeline
  *
@@ -1821,7 +1833,7 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
     let alias = room.getCanonicalAlias();
 
     if (!alias) {
-        const aliases = room.getAliases();
+        const aliases = room.getAltAliases();
 
         if (aliases.length) {
             alias = aliases[0];

src/utils.ts

@@ -734,3 +734,17 @@ export async function promiseMapSeries<T>(
 export function promiseTry<T>(fn: () => T): Promise<T> {
     return new Promise((resolve) => resolve(fn()));
 }
+
+// We need to be able to access the Node.js crypto library from within the
+// Matrix SDK without needing to `require("crypto")`, which will fail in
+// browsers.  So `index.ts` will call `setCrypto` to store it, and when we need
+// it, we can call `getCrypto`.
+let crypto: Object;
+
+export function setCrypto(c: Object) {
+    crypto = c;
+}
+
+export function getCrypto(): Object {
+    return crypto;
+}