v8.3.0

Prepare changelog for v8.3.0
v8.3.0-rc.1
2020-09-14 13:26:27 +01:00 · 2020-09-14 13:26:27 +01:00 · 2020-09-09 15:06:48 +01:00 · 2020-09-09 15:06:47 +01:00 · 2020-09-03 15:51:25 +01:00 · 2020-09-02 17:30:16 +01:00
82 changed files with 6156 additions and 2731 deletions
@@ -1,71 +1,15 @@
 module.exports = {
-    parser: "babel-eslint", // now needed for class properties
-    parserOptions: {
-        sourceType: "module",
-        ecmaFeatures: {
-        }
-    },
+    extends: ["matrix-org"],
+    plugins: [
+        "babel",
+    ],
    env: {
        browser: true,
        node: true,
-
-        // babel's transform-runtime converts references to ES6 globals such as
-        // Promise and Map to core-js polyfills, so we can use ES6 globals.
-        es6: true,
-        jest: true,
    },
-    extends: ["eslint:recommended", "google"],
-    plugins: [
-        "babel",
-        "jest",
-    ],
+
    rules: {
-        // rules we've always adhered to or now do
-        "max-len": ["error", {
-            code: 90,
-            ignoreComments: true,
-        }],
-        curly: ["error", "multi-line"],
-        "prefer-const": ["error"],
-        "comma-dangle": ["error", {
-            arrays: "always-multiline",
-            objects: "always-multiline",
-            imports: "always-multiline",
-            exports: "always-multiline",
-            functions: "always-multiline",
-        }],
-
-        // loosen jsdoc requirements a little
-        "require-jsdoc": ["error", {
-            require: {
-                FunctionDeclaration: false,
-            }
-        }],
-        "valid-jsdoc": ["error", {
-            requireParamDescription: false,
-            requireReturn: false,
-            requireReturnDescription: false,
-        }],
-
-        // rules we do not want from eslint-recommended
-        "no-console": ["off"],
-        "no-constant-condition": ["off"],
-        "no-empty": ["error", { "allowEmptyCatch": true }],
-
-        // rules we do not want from the google styleguide
-        "object-curly-spacing": ["off"],
-        "spaced-comment": ["off"],
-        "guard-for-in": ["off"],
-
-        // in principle we prefer single quotes, but life is too short
-        quotes: ["off"],
-
-        // rules we'd ideally like to adhere to, but the current
-        // code does not (in most cases because it's still ES5)
-        // we set these to warnings, and assert that the number
-        // of warnings doesn't exceed a given threshold
        "no-var": ["warn"],
-        "brace-style": ["warn", "1tbs", {"allowSingleLine": true}],
        "prefer-rest-params": ["warn"],
        "prefer-spread": ["warn"],
        "one-var": ["warn"],
@@ -79,10 +23,19 @@ module.exports = {
            "asyncArrow": "always",
        }],
        "arrow-parens": "off",
-
-        // eslint's built in no-invalid-this rule breaks with class properties
-        "no-invalid-this": "off",
-        // so we replace it with a version that is class property aware
-        "babel/no-invalid-this": "error",
-    }
-}
+        "prefer-promise-reject-errors": "off",
+        "quotes": "off",
+        "indent": "off",
+        "no-constant-condition": "off",
+        "no-async-promise-executor": "off",
+    },
+    overrides: [{
+        "files": ["src/**/*.ts"],
+        "extends": ["matrix-org/ts"],
+        "rules": {
+            // While we're converting to ts we make heavy use of this
+            "@typescript-eslint/no-explicit-any": "off",
+            "quotes": "off",
+        },
+    }],
+};
@@ -1,3 +1,452 @@
+Changes in [8.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.3.0) (2020-09-14)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.3.0-rc.1...v8.3.0)
+
+* No changes since rc.1
+
+Changes in [8.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.3.0-rc.1) (2020-09-09)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.2.0...v8.3.0-rc.1)
+
+ * Add missing options in ICreateClientOpts
+   [\#1452](https://github.com/matrix-org/matrix-js-sdk/pull/1452)
+ * Ensure ready functions return boolean values
+   [\#1457](https://github.com/matrix-org/matrix-js-sdk/pull/1457)
+ * Handle missing cross-signing keys gracefully
+   [\#1456](https://github.com/matrix-org/matrix-js-sdk/pull/1456)
+ * Fix eslint ts override tsx matching
+   [\#1451](https://github.com/matrix-org/matrix-js-sdk/pull/1451)
+ * Untangle cross-signing and secret storage
+   [\#1450](https://github.com/matrix-org/matrix-js-sdk/pull/1450)
+
+Changes in [8.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.2.0) (2020-09-01)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.2.0-rc.1...v8.2.0)
+
+## Security notice
+
+JS SDK 8.2.0 fixes an issue where encrypted state events could break incoming call handling.
+Thanks to @awesome-michael from Awesome Technologies for responsibly disclosing this via Matrix's
+Security Disclosure Policy.
+
+## All changes
+
+* No changes since rc.1
+
+Changes in [8.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.2.0-rc.1) (2020-08-26)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.1.0...v8.2.0-rc.1)
+
+ * Add state event check
+   [\#1449](https://github.com/matrix-org/matrix-js-sdk/pull/1449)
+ * Add method to check whether client .well-known has been fetched
+   [\#1444](https://github.com/matrix-org/matrix-js-sdk/pull/1444)
+ * Handle auth errors during cross-signing key upload
+   [\#1443](https://github.com/matrix-org/matrix-js-sdk/pull/1443)
+ * Don't fail if the requested audio output isn't available
+   [\#1448](https://github.com/matrix-org/matrix-js-sdk/pull/1448)
+ * Fix logging failures
+   [\#1447](https://github.com/matrix-org/matrix-js-sdk/pull/1447)
+ * Log the constraints we pass to getUserMedia
+   [\#1446](https://github.com/matrix-org/matrix-js-sdk/pull/1446)
+ * Use SAS emoji data from matrix-doc
+   [\#1440](https://github.com/matrix-org/matrix-js-sdk/pull/1440)
+
+Changes in [8.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.1.0) (2020-08-17)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.1.0-rc.1...v8.1.0)
+
+* No changes since rc.1
+
+Changes in [8.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.1.0-rc.1) (2020-08-13)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.1...v8.1.0-rc.1)
+
+ * Update on Promises
+   [\#1438](https://github.com/matrix-org/matrix-js-sdk/pull/1438)
+ * Store and request master cross-signing key
+   [\#1437](https://github.com/matrix-org/matrix-js-sdk/pull/1437)
+ * Filter out non-string display names
+   [\#1433](https://github.com/matrix-org/matrix-js-sdk/pull/1433)
+ * Bump elliptic from 6.5.2 to 6.5.3
+   [\#1427](https://github.com/matrix-org/matrix-js-sdk/pull/1427)
+ * Replace Riot with Element in docs and comments
+   [\#1431](https://github.com/matrix-org/matrix-js-sdk/pull/1431)
+ * Remove leftover bits of TSLint
+   [\#1430](https://github.com/matrix-org/matrix-js-sdk/pull/1430)
+
+Changes in [8.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.1) (2020-08-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.1-rc.1...v8.0.1)
+
+ * Filter out non-string display names
+   [\#1434](https://github.com/matrix-org/matrix-js-sdk/pull/1434)
+
+Changes in [8.0.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.1-rc.1) (2020-07-31)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v8.0.0...v8.0.1-rc.1)
+
+ * Remove redundant lint dependencies
+   [\#1426](https://github.com/matrix-org/matrix-js-sdk/pull/1426)
+ * Upload all keys when we start using a new key backup version
+   [\#1428](https://github.com/matrix-org/matrix-js-sdk/pull/1428)
+ * Expose countSessionsNeedingBackup
+   [\#1429](https://github.com/matrix-org/matrix-js-sdk/pull/1429)
+ * Configure and use new eslint package
+   [\#1422](https://github.com/matrix-org/matrix-js-sdk/pull/1422)
+
+Changes in [8.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v8.0.0) (2020-07-27)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.1.0...v8.0.0)
+
+BREAKING CHANGES
+---
+
+* `RoomState` events changed to use a Map instead of an object, which changes the collection APIs available to access them.
+ 
+All Changes
+---
+
+ * Properly support txnId
+   [\#1424](https://github.com/matrix-org/matrix-js-sdk/pull/1424)
+ * [BREAKING] Remove deprecated getIdenticonUri
+   [\#1423](https://github.com/matrix-org/matrix-js-sdk/pull/1423)
+ * Bump lodash from 4.17.15 to 4.17.19
+   [\#1421](https://github.com/matrix-org/matrix-js-sdk/pull/1421)
+ * [BREAKING] Convert RoomState's stored state map to a real map
+   [\#1419](https://github.com/matrix-org/matrix-js-sdk/pull/1419)
+
+Changes in [7.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.1.0) (2020-07-03)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.1.0-rc.1...v7.1.0)
+
+* No changes since rc.1
+
+Changes in [7.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.1.0-rc.1) (2020-07-01)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.0.0...v7.1.0-rc.1)
+
+ * Ask general crypto callbacks for 4S privkey if operation adapter doesn't
+   have it yet
+   [\#1414](https://github.com/matrix-org/matrix-js-sdk/pull/1414)
+ * Fix ICreateClientOpts missing idBaseUrl
+   [\#1413](https://github.com/matrix-org/matrix-js-sdk/pull/1413)
+ * Increase max event listeners for rooms
+   [\#1411](https://github.com/matrix-org/matrix-js-sdk/pull/1411)
+ * Don't trust keys megolm received from backup for verifying the sender
+   [\#1406](https://github.com/matrix-org/matrix-js-sdk/pull/1406)
+ * Raise the last known account data / state event for an update
+   [\#1410](https://github.com/matrix-org/matrix-js-sdk/pull/1410)
+ * Isolate encryption bootstrap side-effects
+   [\#1380](https://github.com/matrix-org/matrix-js-sdk/pull/1380)
+ * Add method to get current in-flight to-device requests
+   [\#1405](https://github.com/matrix-org/matrix-js-sdk/pull/1405)
+
+Changes in [7.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.0.0) (2020-06-23)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v7.0.0-rc.1...v7.0.0)
+
+* No changes since rc.1
+
+Changes in [7.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v7.0.0-rc.1) (2020-06-17)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.2...v7.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * Presence lists were removed from the spec in r0.5.0, and the corresponding methods have now been removed here as well:
+   * `getPresenceList`
+   * `inviteToPresenceList`
+   * `dropFromPresenceList`
+
+All changes
+---
+
+ * Remove support for unspecced device-specific push rules
+   [\#1404](https://github.com/matrix-org/matrix-js-sdk/pull/1404)
+ * Use existing session id for fetching flows as to not get a new session
+   [\#1403](https://github.com/matrix-org/matrix-js-sdk/pull/1403)
+ * Upgrade deps
+   [\#1400](https://github.com/matrix-org/matrix-js-sdk/pull/1400)
+ * Bring back backup key format migration
+   [\#1398](https://github.com/matrix-org/matrix-js-sdk/pull/1398)
+ * Fix: more informative error message when we cant find a key to decrypt with
+   [\#1313](https://github.com/matrix-org/matrix-js-sdk/pull/1313)
+ * Add js-sdk mechanism for polling client well-known for config
+   [\#1394](https://github.com/matrix-org/matrix-js-sdk/pull/1394)
+ * Fix verification request timeouts to match spec
+   [\#1388](https://github.com/matrix-org/matrix-js-sdk/pull/1388)
+ * Drop presence list methods
+   [\#1391](https://github.com/matrix-org/matrix-js-sdk/pull/1391)
+ * Batch up URL previews to prevent excessive requests
+   [\#1395](https://github.com/matrix-org/matrix-js-sdk/pull/1395)
+
+Changes in [6.2.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.2) (2020-06-16)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.1...v6.2.2)
+
+ * Use existing session id for fetching flows as to not get a new session
+   [\#1407](https://github.com/matrix-org/matrix-js-sdk/pull/1407)
+
+Changes in [6.2.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.1) (2020-06-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.0...v6.2.1)
+
+ * Bring back backup key format migration
+   [\#1399](https://github.com/matrix-org/matrix-js-sdk/pull/1399)
+
+Changes in [6.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.0) (2020-06-04)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.2.0-rc.1...v6.2.0)
+
+ * No changes since rc.1
+
+Changes in [6.2.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.2.0-rc.1) (2020-06-02)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.1.0...v6.2.0-rc.1)
+
+ * Make auth argument in the register request compliant with r0.6.0
+   [\#1304](https://github.com/matrix-org/matrix-js-sdk/pull/1304)
+ * Send the wrong auth params with the right auth params
+   [\#1393](https://github.com/matrix-org/matrix-js-sdk/pull/1393)
+ * encrypt cached keys with pickle key
+   [\#1387](https://github.com/matrix-org/matrix-js-sdk/pull/1387)
+ * Fix replying to key share requests
+   [\#1385](https://github.com/matrix-org/matrix-js-sdk/pull/1385)
+ * Add dist to package.json files so CDNs can serve it
+   [\#1384](https://github.com/matrix-org/matrix-js-sdk/pull/1384)
+ * Fix getVersion warning saying undefined room
+   [\#1382](https://github.com/matrix-org/matrix-js-sdk/pull/1382)
+ * Combine the two places we processed client-level default push rules
+   [\#1379](https://github.com/matrix-org/matrix-js-sdk/pull/1379)
+ * make MAC check robust against unpadded vs padded base64 differences
+   [\#1378](https://github.com/matrix-org/matrix-js-sdk/pull/1378)
+ * Remove key backup format migration
+   [\#1375](https://github.com/matrix-org/matrix-js-sdk/pull/1375)
+ * Add simple browserify browser-matrix.js tests
+   [\#1241](https://github.com/matrix-org/matrix-js-sdk/pull/1241)
+ * support new key agreement method for SAS
+   [\#1376](https://github.com/matrix-org/matrix-js-sdk/pull/1376)
+
+Changes in [6.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.1.0) (2020-05-19)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.1.0-rc.1...v6.1.0)
+
+ * No changes since rc.1
+
+Changes in [6.1.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.1.0-rc.1) (2020-05-14)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0...v6.1.0-rc.1)
+
+ * Remove support for asymmetric 4S encryption
+   [\#1373](https://github.com/matrix-org/matrix-js-sdk/pull/1373)
+ * Increase timeout for 2nd phase of Olm session creation
+   [\#1367](https://github.com/matrix-org/matrix-js-sdk/pull/1367)
+ * Add logging on decryption retries
+   [\#1366](https://github.com/matrix-org/matrix-js-sdk/pull/1366)
+ * Emit event when a trusted self-key is stored
+   [\#1364](https://github.com/matrix-org/matrix-js-sdk/pull/1364)
+ * Customize error payload for oversized messages
+   [\#1352](https://github.com/matrix-org/matrix-js-sdk/pull/1352)
+ * Return null for key backup state when we haven't checked yet
+   [\#1363](https://github.com/matrix-org/matrix-js-sdk/pull/1363)
+ * Added a progressCallback for backup key loading
+   [\#1351](https://github.com/matrix-org/matrix-js-sdk/pull/1351)
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1360](https://github.com/matrix-org/matrix-js-sdk/pull/1360)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1359](https://github.com/matrix-org/matrix-js-sdk/pull/1359)
+ * Handle race between sending and await next event from other party
+   [\#1357](https://github.com/matrix-org/matrix-js-sdk/pull/1357)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1354](https://github.com/matrix-org/matrix-js-sdk/pull/1354)
+ * Fix sender of local echo events in unsigned redactions
+   [\#1350](https://github.com/matrix-org/matrix-js-sdk/pull/1350)
+ * Remove redundant key backup setup path
+   [\#1353](https://github.com/matrix-org/matrix-js-sdk/pull/1353)
+ * Remove some dead code from _retryDecryption
+   [\#1349](https://github.com/matrix-org/matrix-js-sdk/pull/1349)
+ * Don't send key requests until after sync processing is finished
+   [\#1348](https://github.com/matrix-org/matrix-js-sdk/pull/1348)
+ * Prevent attempts to send olm messages to ourselves
+   [\#1346](https://github.com/matrix-org/matrix-js-sdk/pull/1346)
+ * Retry account data upload requests
+   [\#1345](https://github.com/matrix-org/matrix-js-sdk/pull/1345)
+ * Log first known index with megolm session updates
+   [\#1344](https://github.com/matrix-org/matrix-js-sdk/pull/1344)
+ * Prune to_device messages to avoid sending empty messages
+   [\#1343](https://github.com/matrix-org/matrix-js-sdk/pull/1343)
+ * Convert bunch of things to TypeScript
+   [\#1335](https://github.com/matrix-org/matrix-js-sdk/pull/1335)
+ * Add logging when making new Olm sessions
+   [\#1342](https://github.com/matrix-org/matrix-js-sdk/pull/1342)
+ * Fix: handle filter not found
+   [\#1340](https://github.com/matrix-org/matrix-js-sdk/pull/1340)
+ * Make getAccountDataFromServer return null if not found
+   [\#1338](https://github.com/matrix-org/matrix-js-sdk/pull/1338)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1336](https://github.com/matrix-org/matrix-js-sdk/pull/1336)
+ * Document setRoomEncryption not modifying room state
+   [\#1328](https://github.com/matrix-org/matrix-js-sdk/pull/1328)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1332](https://github.com/matrix-org/matrix-js-sdk/pull/1332)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1329](https://github.com/matrix-org/matrix-js-sdk/pull/1329)
+ * Fix FilterComponent allowed_values check
+   [\#1327](https://github.com/matrix-org/matrix-js-sdk/pull/1327)
+ * Serialise Olm prekey decryptions
+   [\#1326](https://github.com/matrix-org/matrix-js-sdk/pull/1326)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1324](https://github.com/matrix-org/matrix-js-sdk/pull/1324)
+ * Fix cross-signing/SSSS reset
+   [\#1322](https://github.com/matrix-org/matrix-js-sdk/pull/1322)
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1320](https://github.com/matrix-org/matrix-js-sdk/pull/1320)
+
+Changes in [6.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0) (2020-05-05)
+================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.2...v6.0.0)
+
+ * Add progress callback for key backups
+   [\#1368](https://github.com/matrix-org/matrix-js-sdk/pull/1368)
+
+Changes in [6.0.0-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.2) (2020-05-01)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v6.0.0-rc.1...v6.0.0-rc.2)
+
+ * Emit event when a trusted self-key is stored
+   [\#1365](https://github.com/matrix-org/matrix-js-sdk/pull/1365)
+
+Changes in [6.0.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v6.0.0-rc.1) (2020-04-30)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.4...v6.0.0-rc.1)
+
+BREAKING CHANGES
+---
+
+ * client.getStoredDevicesForUser and client.getStoredDevices are no longer async
+
+All Changes
+---
+
+ * Add initialFetch param to willUpdateDevices / devicesUpdated
+   [\#1362](https://github.com/matrix-org/matrix-js-sdk/pull/1362)
+ * Fix race between sending .request and receiving .ready over to_device
+   [\#1361](https://github.com/matrix-org/matrix-js-sdk/pull/1361)
+ * Handle race between sending and await next event from other party
+   [\#1358](https://github.com/matrix-org/matrix-js-sdk/pull/1358)
+ * Add crypto.willUpdateDevices event and make
+   getStoredDevices/getStoredDevicesForUser synchronous
+   [\#1356](https://github.com/matrix-org/matrix-js-sdk/pull/1356)
+ * Remove redundant key backup setup path
+   [\#1355](https://github.com/matrix-org/matrix-js-sdk/pull/1355)
+
+Changes in [5.3.1-rc.4](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.4) (2020-04-23)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.3...v5.3.1-rc.4)
+
+ * Retry account data upload requests
+   [\#1347](https://github.com/matrix-org/matrix-js-sdk/pull/1347)
+ * Fix: handle filter not found
+   [\#1341](https://github.com/matrix-org/matrix-js-sdk/pull/1341)
+ * Make getAccountDataFromServer return null if not found
+   [\#1339](https://github.com/matrix-org/matrix-js-sdk/pull/1339)
+ * Fix setDefaultKeyId to fail if the request fails
+   [\#1337](https://github.com/matrix-org/matrix-js-sdk/pull/1337)
+ * Fix: don't do extra /filter request when enabling lazy loading of members
+   [\#1333](https://github.com/matrix-org/matrix-js-sdk/pull/1333)
+ * Reject attemptAuth promise if no auth flow found
+   [\#1331](https://github.com/matrix-org/matrix-js-sdk/pull/1331)
+ * Serialise Olm prekey decryptions
+   [\#1330](https://github.com/matrix-org/matrix-js-sdk/pull/1330)
+
+Changes in [5.3.1-rc.3](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.3) (2020-04-17)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.2...v5.3.1-rc.3)
+
+ * Fix cross-signing/SSSS reset
+   [\#1323](https://github.com/matrix-org/matrix-js-sdk/pull/1323)
+ * Fix: crash when backup key needs fixing from corruption issue
+   [\#1325](https://github.com/matrix-org/matrix-js-sdk/pull/1325)
+
+Changes in [5.3.1-rc.2](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.2) (2020-04-16)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.1-rc.1...v5.3.1-rc.2)
+
+ * Implement QR code reciprocate for self-verification with untrusted MSK
+   [\#1321](https://github.com/matrix-org/matrix-js-sdk/pull/1321)
+
+Changes in [5.3.1-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.1-rc.1) (2020-04-15)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.3.0-rc.1...v5.3.1-rc.1)
+
+ * Adapt release script for riot-desktop
+   [\#1319](https://github.com/matrix-org/matrix-js-sdk/pull/1319)
+ * Fix: prevent spurious notifications from indexer
+   [\#1318](https://github.com/matrix-org/matrix-js-sdk/pull/1318)
+ * Always create our own user object
+   [\#1317](https://github.com/matrix-org/matrix-js-sdk/pull/1317)
+ * Fix incorrect backup key format in SSSS
+   [\#1311](https://github.com/matrix-org/matrix-js-sdk/pull/1311)
+ * Fix e2ee crash after refreshing after having received a cross-singing key
+   reset
+   [\#1315](https://github.com/matrix-org/matrix-js-sdk/pull/1315)
+ * Fix: catch send errors in SAS verifier
+   [\#1314](https://github.com/matrix-org/matrix-js-sdk/pull/1314)
+ * Clear cross-signing keys when detecting the keys have changed
+   [\#1312](https://github.com/matrix-org/matrix-js-sdk/pull/1312)
+ * Upgrade deps
+   [\#1310](https://github.com/matrix-org/matrix-js-sdk/pull/1310)
+
+Changes in [5.3.0-rc.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.3.0-rc.1) (2020-04-08)
+==========================================================================================================
+[Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0...v5.3.0-rc.1)
+
+ * Store key backup key in cache as Uint8Array
+   [\#1308](https://github.com/matrix-org/matrix-js-sdk/pull/1308)
+ * Use the correct request body for the /keys/query endpoint.
+   [\#1307](https://github.com/matrix-org/matrix-js-sdk/pull/1307)
+ * Avoid creating two devices on registration
+   [\#1305](https://github.com/matrix-org/matrix-js-sdk/pull/1305)
+ * Lower max-warnings to 81
+   [\#1306](https://github.com/matrix-org/matrix-js-sdk/pull/1306)
+ * Move key backup key creation before caching
+   [\#1303](https://github.com/matrix-org/matrix-js-sdk/pull/1303)
+ * Expose function to force-reset outgoing room key requests
+   [\#1298](https://github.com/matrix-org/matrix-js-sdk/pull/1298)
+ * Add isSelfVerification property to VerificationRequest
+   [\#1302](https://github.com/matrix-org/matrix-js-sdk/pull/1302)
+ * QR code reciprocation
+   [\#1297](https://github.com/matrix-org/matrix-js-sdk/pull/1297)
+ * Add ability to check symmetric SSSS key before we try to use it
+   [\#1294](https://github.com/matrix-org/matrix-js-sdk/pull/1294)
+ * Add some debug logging for events stuck to bottom of timeline
+   [\#1296](https://github.com/matrix-org/matrix-js-sdk/pull/1296)
+ * Fix: spontanous verification request cancellation under some circumstances
+   [\#1295](https://github.com/matrix-org/matrix-js-sdk/pull/1295)
+ * Receive private key for caching from the app layer
+   [\#1293](https://github.com/matrix-org/matrix-js-sdk/pull/1293)
+ * Track whether we have verified a user before
+   [\#1292](https://github.com/matrix-org/matrix-js-sdk/pull/1292)
+ * Fix: error during tests
+   [\#1222](https://github.com/matrix-org/matrix-js-sdk/pull/1222)
+ * Send .done event for to_device verification
+   [\#1288](https://github.com/matrix-org/matrix-js-sdk/pull/1288)
+ * Request the key backup key & restore backup
+   [\#1291](https://github.com/matrix-org/matrix-js-sdk/pull/1291)
+ * Make screen sharing works on Chrome using getDisplayMedia()
+   [\#1276](https://github.com/matrix-org/matrix-js-sdk/pull/1276)
+ * Fix isVerified returning false
+   [\#1289](https://github.com/matrix-org/matrix-js-sdk/pull/1289)
+ * Fix: verification gets cancelled when event gets duplicated
+   [\#1286](https://github.com/matrix-org/matrix-js-sdk/pull/1286)
+ * Use requestSecret on the client to request secrets
+   [\#1287](https://github.com/matrix-org/matrix-js-sdk/pull/1287)
+ * Allow guests to fetch TURN servers
+   [\#1277](https://github.com/matrix-org/matrix-js-sdk/pull/1277)
+
 Changes in [5.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v5.2.0) (2020-03-30)
 ================================================================================================
 [Full Changelog](https://github.com/matrix-org/matrix-js-sdk/compare/v5.2.0-rc.1...v5.2.0)
@@ -182,10 +182,8 @@ you can pass the result of the promise into it with something like:
  matrixClient.someMethod(arg1, arg2).nodeify(callback);
 ```

-The main thing to note is that it is an error to discard the result of a
-promise-returning function, as that will cause exceptions to go unobserved. If
-you have nothing better to do with the result, just call ``.done()`` on it. See
-http://documentup.com/kriskowal/q/#the-end for more information.
+The main thing to note is that it is problematic to discard the result of a
+promise-returning function, as that will cause exceptions to go unobserved.

 Methods which return a promise show this in their documentation.

@@ -288,7 +288,7 @@ function printMemberList(room) {
 }

 function printRoomInfo(room) {
-    var eventDict = room.currentState.events;
+    var eventMap = room.currentState.events;
    var eTypeHeader = "    Event Type(state_key)    ";
    var sendHeader = "        Sender        ";
    // pad content to 100
@@ -300,14 +300,15 @@ function printRoomInfo(room) {
    var contentHeader = padSide + "Content" + padSide;
    print(eTypeHeader+sendHeader+contentHeader);
    print(new Array(100).join("-"));
-    Object.keys(eventDict).forEach(function(eventType) {
+    eventMap.keys().forEach(function(eventType) {
        if (eventType === "m.room.member") { return; } // use /members instead.
-        Object.keys(eventDict[eventType]).forEach(function(stateKey) {
+        var eventEventMap = eventMap.get(eventType);
+        eventEventMap.keys().forEach(function(stateKey) {
            var typeAndKey = eventType + (
                stateKey.length > 0 ? "("+stateKey+")" : ""
            );
            var typeStr = fixWidth(typeAndKey, eTypeHeader.length);
-            var event = eventDict[eventType][stateKey];
+            var event = eventEventMap.get(stateKey);
            var sendStr = fixWidth(event.getSender(), sendHeader.length);
            var contentStr = fixWidth(
                JSON.stringify(event.getContent()), contentHeader.length
@@ -1,6 +1,6 @@
 {
  "name": "matrix-js-sdk",
-  "version": "5.2.0",
+  "version": "8.3.0",
  "description": "Matrix Client-Server SDK for Javascript",
  "scripts": {
    "prepare": "yarn build",
@@ -13,10 +13,9 @@
    "build:compile-browser": "mkdirp dist && browserify -d src/browser-index.js -p [ tsify -p ./tsconfig.json ] -t [ babelify --sourceMaps=inline --presets [ @babel/preset-env @babel/preset-typescript ] ] | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js",
    "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
    "gendoc": "jsdoc -c jsdoc.json -P package.json",
-    "lint": "yarn lint:types && yarn lint:ts && yarn lint:js",
-    "lint:js": "eslint --max-warnings 93 src spec",
+    "lint": "yarn lint:types && yarn lint:js",
+    "lint:js": "eslint --max-warnings 76 src spec",
    "lint:types": "tsc --noEmit",
-    "lint:ts": "tslint --project ./tsconfig.json -t stylish",
    "test": "jest spec/ --coverage --testEnvironment node",
    "test:watch": "jest spec/ --coverage --testEnvironment node --watch"
  },
@@ -35,6 +34,7 @@
  "author": "matrix.org",
  "license": "Apache-2.0",
  "files": [
+    "dist",
    "lib",
    "src",
    "git-revision.txt",
@@ -68,15 +68,15 @@
    "@babel/preset-typescript": "^7.7.4",
    "@babel/register": "^7.7.4",
    "@types/node": "12",
+    "@types/request": "^2.48.4",
    "babel-eslint": "^10.0.3",
    "babel-jest": "^24.9.0",
    "babelify": "^10.0.0",
    "better-docs": "^1.4.7",
    "browserify": "^16.5.0",
-    "eslint": "^5.12.0",
-    "eslint-config-google": "^0.7.1",
+    "eslint": "7.3.1",
+    "eslint-config-matrix-org": "^0.1.2",
    "eslint-plugin-babel": "^5.3.0",
-    "eslint-plugin-jest": "^23.0.4",
    "exorcist": "^1.0.1",
    "fake-indexeddb": "^3.0.0",
    "jest": "^24.9.0",
@@ -87,7 +87,6 @@
    "rimraf": "^3.0.0",
    "terser": "^4.4.3",
    "tsify": "^4.0.1",
-    "tslint": "^5.20.1",
    "typescript": "^3.7.3"
  },
  "jest": {
@@ -10,7 +10,7 @@
 #   npm; typically installed by Node.js
 #   yarn; install via brew (macOS) or similar (https://yarnpkg.com/docs/install/)
 #
-# Note: this script is also used to release matrix-react-sdk and riot-web.
+# Note: this script is also used to release matrix-react-sdk and element-web.

 set -e

@@ -38,6 +38,7 @@ $USAGE
    -c changelog_file:  specify name of file containing changelog
    -x:                 skip updating the changelog
    -z:                 skip generating the jsdoc
+    -n:                 skip publish to NPM
 EOF
 }

@@ -60,9 +61,10 @@ fi

 skip_changelog=
 skip_jsdoc=
+skip_npm=
 changelog_file="CHANGELOG.md"
 expected_npm_user="matrixdotorg"
-while getopts hc:u:xz f; do
+while getopts hc:u:xzn f; do
    case $f in
        h)
            help
@@ -77,6 +79,9 @@ while getopts hc:u:xz f; do
        z)
            skip_jsdoc=1
            ;;
+        n)
+            skip_npm=1
+            ;;
        u)
            expected_npm_user="$OPTARG"
            ;;
@@ -96,10 +101,12 @@ fi

 # Login and publish continues to use `npm`, as it seems to have more clearly
 # defined options and semantics than `yarn` for writing to the registry.
-actual_npm_user=`npm whoami`;
-if [ $expected_npm_user != $actual_npm_user ]; then
-    echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
-    exit 1
+if [ -z "$skip_npm" ]; then
+    actual_npm_user=`npm whoami`;
+    if [ $expected_npm_user != $actual_npm_user ]; then
+        echo "you need to be logged into npm as $expected_npm_user, but you are logged in as $actual_npm_user" >&2
+        exit 1
+    fi
 fi

 # ignore leading v on release
@@ -298,11 +305,13 @@ rm "${latest_changes}"
 # defined options and semantics than `yarn` for writing to the registry.
 # Tag both releases and prereleases as `next` so the last stable release remains
 # the default.
-npm publish --tag next
-if [ $prerelease -eq 0 ]; then
-    # For a release, also add the default `latest` tag.
-    package=$(cat package.json | jq -er .name)
-    npm dist-tag add "$package@$release" latest
+if [ -z "$skip_npm" ]; then
+    npm publish --tag next
+    if [ $prerelease -eq 0 ]; then
+        # For a release, also add the default `latest` tag.
+        package=$(cat package.json | jq -er .name)
+        npm dist-tag add "$package@$release" latest
+    fi
 fi

 if [ -z "$skip_jsdoc" ]; then
@@ -338,8 +347,10 @@ if [ -z "$skip_jsdoc" ]; then
    git push origin gh-pages
 fi

-# finally, merge master back onto develop
-git checkout develop
-git pull
-git merge master
-git push origin develop
+# finally, merge master back onto develop (if it exists)
+if [ $(git branch -lr | grep origin/develop -c) -ge 1 ]; then
+    git checkout develop
+    git pull
+    git merge master
+    git push origin develop
+fi
@@ -185,7 +185,7 @@ TestClient.prototype.expectKeyQuery = function(response) {
        200, (path, content) => {
            Object.keys(response.device_keys).forEach((userId) => {
                expect(content.device_keys[userId]).toEqual(
-                    {},
+                    [],
                    "Expected key query for " + userId + ", got " +
                    Object.keys(content.device_keys),
                );
@@ -0,0 +1,23 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// stub for browser-matrix browserify tests
+global.XMLHttpRequest = jest.fn();
+
+afterAll(() => {
+    // clean up XMLHttpRequest mock
+    global.XMLHttpRequest = undefined;
+});
@@ -0,0 +1,103 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// load XmlHttpRequest mock
+import "./setupTests";
+import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
+import {MockStorageApi} from "../MockStorageApi";
+import {WebStorageSessionStore} from "../../src/store/session/webstorage";
+import MockHttpBackend from "matrix-mock-request";
+import {LocalStorageCryptoStore} from "../../src/crypto/store/localStorage-crypto-store";
+import * as utils from "../test-utils";
+
+const USER_ID = "@user:test.server";
+const DEVICE_ID = "device_id";
+const ACCESS_TOKEN = "access_token";
+const ROOM_ID = "!room_id:server.test";
+
+/* global matrixcs */
+
+describe("Browserify Test", function() {
+    let client;
+    let httpBackend;
+
+    async function createTestClient() {
+        const sessionStoreBackend = new MockStorageApi();
+        const sessionStore = new WebStorageSessionStore(sessionStoreBackend);
+        const httpBackend = new MockHttpBackend();
+
+        const options = {
+            baseUrl: "http://" + USER_ID + ".test.server",
+            userId: USER_ID,
+            accessToken: ACCESS_TOKEN,
+            deviceId: DEVICE_ID,
+            sessionStore: sessionStore,
+            request: httpBackend.requestFn,
+            cryptoStore: new LocalStorageCryptoStore(sessionStoreBackend),
+        };
+
+        const client = matrixcs.createClient(options);
+
+        httpBackend.when("GET", "/pushrules").respond(200, {});
+        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
+
+        return { client, httpBackend };
+    }
+
+    beforeEach(async () => {
+        ({client, httpBackend} = await createTestClient());
+        await client.startClient();
+    });
+
+    afterEach(async () => {
+        client.stopClient();
+        await httpBackend.stop();
+    });
+
+    it("Sync", async function() {
+        const event = utils.mkMembership({
+            room: ROOM_ID,
+            mship: "join",
+            user: "@other_user:server.test",
+            name: "Displayname",
+        });
+
+        const syncData = {
+            next_batch: "batch1",
+            rooms: {
+                join: {},
+            },
+        };
+        syncData.rooms.join[ROOM_ID] = {
+            timeline: {
+                events: [
+                    event,
+                ],
+                limited: false,
+            },
+        };
+
+        httpBackend.when("GET", "/sync").respond(200, syncData);
+        await Promise.race([
+            Promise.all([
+                httpBackend.flushAllExpected(),
+            ]),
+            new Promise((_, reject) => {
+                client.once("sync.unexpectedError", reject);
+            }),
+        ]);
+    }, 10000);
+});
@@ -140,7 +140,7 @@ describe("DeviceList management:", function() {

    it("We should not get confused by out-of-order device query responses",
       () => {
-           // https://github.com/vector-im/riot-web/issues/3126
+           // https://github.com/vector-im/element-web/issues/3126
           aliceTestClient.expectKeyQuery({device_keys: {'@alice:localhost': {}}});
           return aliceTestClient.start().then(() => {
               aliceTestClient.httpBackend.when('GET', '/sync').respond(
@@ -271,7 +271,7 @@ describe("DeviceList management:", function() {
           });
       }).timeout(3000);

-    // https://github.com/vector-im/riot-web/issues/4983
+    // https://github.com/vector-im/element-web/issues/4983
    describe("Alice should know she has stale device lists", () => {
        beforeEach(async function() {
            await aliceTestClient.start();
@@ -70,7 +70,7 @@ function expectAliQueryKeys() {
    aliTestClient.httpBackend.when("POST", "/keys/query")
            .respond(200, function(path, content) {
        expect(content.device_keys[bobUserId]).toEqual(
-            {},
+            [],
            "Expected Alice to key query for " + bobUserId + ", got " +
            Object.keys(content.device_keys),
        );
@@ -98,7 +98,7 @@ function expectBobQueryKeys() {
        "POST", "/keys/query",
    ).respond(200, function(path, content) {
        expect(content.device_keys[aliUserId]).toEqual(
-            {},
+            [],
            "Expected Bob to key query for " + aliUserId + ", got " +
            Object.keys(content.device_keys),
        );
@@ -347,8 +347,8 @@ describe("MatrixClient", function() {

            httpBackend.when("POST", "/keys/query").check(function(req) {
                expect(req.data).toEqual({device_keys: {
-                    'boris': {},
-                    'chaz': {},
+                    'boris': [],
+                    'chaz': [],
                }});
            }).respond(200, {
                device_keys: {
@@ -3,6 +3,7 @@ import HttpBackend from "matrix-mock-request";
 import {MatrixClient} from "../../src/matrix";
 import {MatrixScheduler} from "../../src/scheduler";
 import {MemoryStore} from "../../src/store/memory";
+import {MatrixError} from "../../src/http-api";

 describe("MatrixClient opts", function() {
    const baseUrl = "http://localhost.or.something";
@@ -132,10 +133,10 @@ describe("MatrixClient opts", function() {
        });

        it("shouldn't retry sending events", function(done) {
-            httpBackend.when("PUT", "/txn1").fail(500, {
+            httpBackend.when("PUT", "/txn1").fail(500, new MatrixError({
                errcode: "M_SOMETHING",
                error: "Ruh roh",
-            });
+            }));
            client.sendTextMessage("!foo:bar", "a body", "txn1").then(function(res) {
                expect(false).toBe(true, "sendTextMessage resolved but shouldn't");
            }, function(err) {
@@ -90,7 +90,7 @@ describe("MatrixClient retrying", function() {
        // wait for the localecho of ev1 to be updated
        const p3 = new Promise((resolve, reject) => {
            room.on("Room.localEchoUpdated", (ev0) => {
-                if(ev0 === ev1) {
+                if (ev0 === ev1) {
                    resolve();
                }
            });
@@ -346,7 +346,7 @@ describe("megolm", function() {
    });

    it("Alice receives a megolm message before the session keys", function() {
-        // https://github.com/vector-im/riot-web/issues/2273
+        // https://github.com/vector-im/element-web/issues/2273
        let roomKeyEncrypted;

        return aliceTestClient.start().then(() => {
@@ -726,7 +726,7 @@ describe("megolm", function() {
        });
    });

-    // https://github.com/vector-im/riot-web/issues/2676
+    // https://github.com/vector-im/element-web/issues/2676
    it("Alice should send to her other devices", function() {
        // for this test, we make the testOlmAccount be another of Alice's devices.
        // it ought to get included in messages Alice sends.
@@ -16,6 +16,7 @@ limitations under the License.
 */

 import {logger} from '../src/logger';
+import * as utils from "../src/utils";

 // try to load the olm library.
 try {
@@ -24,3 +25,11 @@ try {
 } catch (e) {
    logger.warn("unable to run crypto tests: libolm not available");
 }
+
+// also try to set node crypto
+try {
+    const crypto = require('crypto');
+    utils.setCrypto(crypto);
+} catch (err) {
+    console.log('nodejs was compiled without crypto support: some tests will fail');
+}
@@ -1,4 +1,4 @@
-import {getHttpUriForMxc, getIdenticonUri} from "../../src/content-repo";
+import {getHttpUriForMxc} from "../../src/content-repo";

 describe("ContentRepo", function() {
    const baseUrl = "https://my.home.server";
@@ -56,31 +56,4 @@ describe("ContentRepo", function() {
            );
        });
    });
-
-    describe("getIdenticonUri", function() {
-        it("should do nothing for null input", function() {
-            expect(getIdenticonUri(null)).toEqual(null);
-        });
-
-        it("should set w/h by default to 96", function() {
-            expect(getIdenticonUri(baseUrl, "foobar")).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=96&height=96",
-            );
-        });
-
-        it("should be able to set custom w/h", function() {
-            expect(getIdenticonUri(baseUrl, "foobar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foobar" +
-                "?width=32&height=64",
-            );
-        });
-
-        it("should URL encode the identicon string", function() {
-            expect(getIdenticonUri(baseUrl, "foo#bar", 32, 64)).toEqual(
-                baseUrl + "/_matrix/media/unstable/identicon/foo%23bar" +
-                "?width=32&height=64",
-            );
-        });
-    });
 });
@@ -10,6 +10,7 @@ import * as olmlib from "../../src/crypto/olmlib";
 import {sleep} from "../../src/utils";
 import {EventEmitter} from "events";
 import {CRYPTO_ENABLED} from "../../src/client";
+import {DeviceInfo} from "../../src/crypto/deviceinfo";

 const Olm = global.Olm;

@@ -26,6 +27,66 @@ describe("Crypto", function() {
        expect(Crypto.getOlmVersion()[0]).toEqual(3);
    });

+    describe("encrypted events", function() {
+        it("provides encryption information", async function() {
+            const client = (new TestClient(
+                "@alice:example.com", "deviceid",
+            )).client;
+            await client.initCrypto();
+
+            // unencrypted event
+            const event = {
+                getId: () => "$event_id",
+                getSenderKey: () => null,
+                getWireContent: () => {return {};},
+            };
+
+            let encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeFalsy();
+
+            // unknown sender (e.g. deleted device), forwarded megolm key (untrusted)
+            event.getSenderKey = () => 'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            event.getWireContent = () => {return {algorithm: olmlib.MEGOLM_ALGORITHM};};
+            event.getForwardingCurve25519KeyChain = () => ["not empty"];
+            event.isKeySourceUntrusted = () => false;
+            event.getClaimedEd25519Key =
+                () => 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeFalsy();
+
+            // known sender, megolm key from backup
+            event.getForwardingCurve25519KeyChain = () => [];
+            event.isKeySourceUntrusted = () => true;
+            const device = new DeviceInfo("FLIBBLE");
+            device.keys["curve25519:FLIBBLE"] =
+                'YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI';
+            device.keys["ed25519:FLIBBLE"] =
+                'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA';
+            client._crypto._deviceList.getDeviceByIdentityKey = () => device;
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeFalsy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeFalsy();
+
+            // known sender, trusted megolm key, but bad ed25519key
+            event.isKeySourceUntrusted = () => false;
+            device.keys["ed25519:FLIBBLE"] =
+                'BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB';
+
+            encryptionInfo = client.getEventEncryptionInfo(event);
+            expect(encryptionInfo.encrypted).toBeTruthy();
+            expect(encryptionInfo.authenticated).toBeTruthy();
+            expect(encryptionInfo.sender).toBeTruthy();
+            expect(encryptionInfo.mismatchedSender).toBeTruthy();
+
+            client.stopClient();
+        });
+    });

    describe('Session management', function() {
        const otkResponse = {
@@ -313,6 +374,10 @@ describe("Crypto", function() {
            // make a room key request, and record the transaction ID for the
            // sendToDevice call
            await aliceClient.cancelAndResendEventRoomKeyRequest(event);
+            // key requests get queued until the sync has finished, but we don't
+            // let the client set up enough for that to happen, so gut-wrench a bit
+            // to force it to send now.
+            aliceClient._crypto._outgoingRoomKeyRequestManager.sendQueuedRequests();
            jest.runAllTimers();
            await Promise.resolve();
            expect(aliceClient.sendToDevice).toBeCalledTimes(1);
@@ -25,6 +25,7 @@ import {
 import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
 import 'fake-indexeddb/auto';
 import 'jest-localstorage-mock';
+import {OlmDevice} from "../../../src/crypto/OlmDevice";

 const userId = "@alice:example.com";

@@ -37,7 +38,7 @@ const testKey = new Uint8Array([
 ]);

 const types = [
-    { type: "master", shouldCache: false },
+    { type: "master", shouldCache: true },
    { type: "self_signing", shouldCache: true },
    { type: "user_signing", shouldCache: true },
    { type: "invalid", shouldCache: false },
@@ -233,8 +234,9 @@ describe.each([

    it("should cache data to the store and retrieve it", async () => {
        await store.startup();
+        const olmDevice = new OlmDevice(store);
        const { getCrossSigningKeyCache, storeCrossSigningKeyCache } =
-          createCryptoStoreCacheCallbacks(store);
+              createCryptoStoreCacheCallbacks(store, olmDevice);
        await storeCrossSigningKeyCache("self_signing", testKey);

        // If we've not saved anything, don't expect anything
@@ -243,6 +245,6 @@ describe.each([
        expect(nokey).toBeNull();

        const key = await getCrossSigningKeyCache("self_signing", "");
-        expect(key).toEqual(testKey);
+        expect(new Uint8Array(key)).toEqual(testKey);
    });
 });
@@ -27,6 +27,7 @@ import {MockStorageApi} from "../../MockStorageApi";
 import * as testUtils from "../../test-utils";
 import {OlmDevice} from "../../../src/crypto/OlmDevice";
 import {Crypto} from "../../../src/crypto";
+import {resetCrossSigningKeys} from "./crypto-utils";

 const Olm = global.Olm;

@@ -332,7 +333,7 @@ describe("MegolmBackup", function() {
            client.on("crossSigning.getKey", function(e) {
                e.done(privateKeys[e.type]);
            });
-            await client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(client);
            let numCalls = 0;
            await new Promise((resolve, reject) => {
                client._http.authedRequest = function(
@@ -517,6 +518,7 @@ describe("MegolmBackup", function() {
                return megolmDecryption.decryptEvent(ENCRYPTED_EVENT);
            }).then((res) => {
                expect(res.clearEvent.content).toEqual('testytest');
+                expect(res.untrusted).toBeTruthy(); // keys from backup are untrusted
            });
        });

@@ -546,7 +548,7 @@ describe("MegolmBackup", function() {
            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
            await client._crypto.storeSessionBackupPrivateKey(key);
            const result = await client._crypto.getSessionBackupPrivateKey();
-            expect(result).toEqual(key);
+            expect(new Uint8Array(result)).toEqual(key);
        });

        it('caches session backup keys as it encounters them', async function() {
@@ -20,6 +20,8 @@ import anotherjson from 'another-json';
 import * as olmlib from "../../../src/crypto/olmlib";
 import {TestClient} from '../../TestClient';
 import {HttpResponse, setHttpResponses} from '../../test-utils';
+import { resetCrossSigningKeys } from "./crypto-utils";
+import { MatrixError } from '../../../src/http-api';

 async function makeTestClient(userInfo, options, keys) {
    if (!keys) keys = {};
@@ -66,11 +68,66 @@ describe("Cross Signing", function() {
            );
        });
        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => {};
        // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await alice.bootstrapCrossSigning({
+            authUploadDeviceSigningKeys: async func => await func({}),
+        });
        expect(alice.uploadDeviceSigningKeys).toHaveBeenCalled();
    });

+    it("should abort bootstrap if device signing auth fails", async function() {
+        const alice = await makeTestClient(
+            {userId: "@alice:example.com", deviceId: "Osborne2"},
+        );
+        alice.uploadDeviceSigningKeys = async (auth, keys) => {
+            const errorResponse = {
+                session: "sessionId",
+                flows: [
+                    {
+                        stages: [
+                            "m.login.password",
+                        ],
+                    },
+                ],
+                params: {},
+            };
+
+            // If we're not just polling for flows, add on error rejecting the
+            // auth attempt.
+            if (auth) {
+                Object.assign(errorResponse, {
+                    completed: [],
+                    error: "Invalid password",
+                    errcode: "M_FORBIDDEN",
+                });
+            }
+
+            const error = new MatrixError(errorResponse);
+            error.httpStatus == 401;
+            throw error;
+        };
+        alice.uploadKeySignatures = async () => {};
+        alice.setAccountData = async () => {};
+        alice.getAccountDataFromServer = async () => { };
+        const authUploadDeviceSigningKeys = async func => await func({});
+
+        // Try bootstrap, expecting `authUploadDeviceSigningKeys` to pass
+        // through failure, stopping before actually applying changes.
+        let bootstrapDidThrow = false;
+        try {
+            await alice.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys,
+            });
+        } catch (e) {
+            if (e.errcode === "M_FORBIDDEN") {
+                bootstrapDidThrow = true;
+            }
+        }
+        expect(bootstrapDidThrow).toBeTruthy();
+    });
+
    it("should upload a signature when a user is verified", async function() {
        const alice = await makeTestClient(
            {userId: "@alice:example.com", deviceId: "Osborne2"},
@@ -78,7 +135,7 @@ describe("Cross Signing", function() {
        alice.uploadDeviceSigningKeys = async () => {};
        alice.uploadKeySignatures = async () => {};
        // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
        // Alice downloads Bob's device key
        alice._crypto._deviceList.storeCrossSigningForUser("@bob:example.com", {
            keys: {
@@ -273,7 +330,7 @@ describe("Cross Signing", function() {
        alice.uploadDeviceSigningKeys = async () => {};
        alice.uploadKeySignatures = async () => {};
        // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
        // Alice downloads Bob's ssk and device key
        const bobMasterSigning = new global.Olm.PkSigning();
        const bobMasterPrivkey = bobMasterSigning.generate_seed();
@@ -363,7 +420,7 @@ describe("Cross Signing", function() {
        alice.uploadKeySignatures = async () => {};

        // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);

        const selfSigningKey = new Uint8Array([
            0x1e, 0xf4, 0x01, 0x6d, 0x4f, 0xa1, 0x73, 0x66,
@@ -520,7 +577,7 @@ describe("Cross Signing", function() {
        alice.uploadDeviceSigningKeys = async () => {};
        alice.uploadKeySignatures = async () => {};
        // set Alice's cross-signing key
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
        // Alice downloads Bob's ssk and device key
        // (NOTE: device key is not signed by ssk)
        const bobMasterSigning = new global.Olm.PkSigning();
@@ -588,7 +645,7 @@ describe("Cross Signing", function() {
        );
        alice.uploadDeviceSigningKeys = async () => {};
        alice.uploadKeySignatures = async () => {};
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
        // Alice downloads Bob's keys
        const bobMasterSigning = new global.Olm.PkSigning();
        const bobMasterPrivkey = bobMasterSigning.generate_seed();
@@ -740,7 +797,7 @@ describe("Cross Signing", function() {
        bob.uploadDeviceSigningKeys = async () => {};
        bob.uploadKeySignatures = async () => {};
        // set Bob's cross-signing key
-        await bob.resetCrossSigningKeys();
+        await resetCrossSigningKeys(bob);
        alice._crypto._deviceList.storeDevicesForUser("@bob:example.com", {
            Dynabook: {
                algorithms: ["m.olm.curve25519-aes-sha256", "m.megolm.v1.aes-sha"],
@@ -766,7 +823,7 @@ describe("Cross Signing", function() {
        let upgradePromise = new Promise((resolve) => {
            upgradeResolveFunc = resolve;
        });
-        await alice.resetCrossSigningKeys();
+        await resetCrossSigningKeys(alice);
        await upgradePromise;

        const bobTrust = alice.checkUserTrust("@bob:example.com");
@@ -0,0 +1,44 @@
+import {IndexedDBCryptoStore} from '../../../src/crypto/store/indexeddb-crypto-store';
+
+
+// needs to be phased out and replaced with bootstrapSecretStorage,
+// but that is doing too much extra stuff for it to be an easy transition.
+export async function resetCrossSigningKeys(client, {
+    level,
+    authUploadDeviceSigningKeys = async func => await func(),
+} = {}) {
+    const crypto = client._crypto;
+
+    const oldKeys = Object.assign({}, crypto._crossSigningInfo.keys);
+    try {
+        await crypto._crossSigningInfo.resetKeys(level);
+        await crypto._signObject(crypto._crossSigningInfo.keys.master);
+        // write a copy locally so we know these are trusted keys
+        await crypto._cryptoStore.doTxn(
+            'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+            (txn) => {
+                crypto._cryptoStore.storeCrossSigningKeys(
+                    txn, crypto._crossSigningInfo.keys);
+            },
+        );
+    } catch (e) {
+        // If anything failed here, revert the keys so we know to try again from the start
+        // next time.
+        crypto._crossSigningInfo.keys = oldKeys;
+        throw e;
+    }
+    crypto._baseApis.emit("crossSigning.keysChanged", {});
+    await crypto._afterCrossSigningLocalKeyChange();
+}
+
+export async function createSecretStorageKey() {
+    const decryption = new global.Olm.PkDecryption();
+    const storagePublicKey = decryption.generate_key();
+    const storagePrivateKey = decryption.get_private_key();
+    decryption.free();
+    return {
+        // `pubkey` not used anymore with symmetric 4S
+        keyInfo: { pubkey: storagePublicKey },
+        privateKey: storagePrivateKey,
+    };
+}
@@ -0,0 +1,87 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    IndexedDBCryptoStore,
+} from '../../../src/crypto/store/indexeddb-crypto-store';
+import {MemoryCryptoStore} from '../../../src/crypto/store/memory-crypto-store';
+import 'fake-indexeddb/auto';
+import 'jest-localstorage-mock';
+
+import {
+    ROOM_KEY_REQUEST_STATES,
+} from '../../../src/crypto/OutgoingRoomKeyRequestManager';
+
+const requests = [
+    {
+        requestId: "A",
+        requestBody: { session_id: "A", room_id: "A" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "B",
+        requestBody: { session_id: "B", room_id: "B" },
+        state: ROOM_KEY_REQUEST_STATES.SENT,
+    },
+    {
+        requestId: "C",
+        requestBody: { session_id: "C", room_id: "C" },
+        state: ROOM_KEY_REQUEST_STATES.UNSENT,
+    },
+];
+
+describe.each([
+    ["IndexedDBCryptoStore",
+     () => new IndexedDBCryptoStore(global.indexedDB, "tests")],
+    ["LocalStorageCryptoStore",
+     () => new IndexedDBCryptoStore(undefined, "tests")],
+    ["MemoryCryptoStore", () => {
+        const store = new IndexedDBCryptoStore(undefined, "tests");
+        store._backend = new MemoryCryptoStore();
+        store._backendPromise = Promise.resolve(store._backend);
+        return store;
+    }],
+])("Outgoing room key requests [%s]", function(name, dbFactory) {
+    let store;
+
+    beforeAll(async () => {
+        store = dbFactory();
+        await store.startup();
+        await Promise.all(requests.map((request) =>
+            store.getOrAddOutgoingRoomKeyRequest(request),
+        ));
+    });
+
+    it("getAllOutgoingRoomKeyRequestsByState retrieves all entries in a given state",
+    async () => {
+        const r = await
+            store.getAllOutgoingRoomKeyRequestsByState(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(r).toHaveLength(2);
+        requests.filter((e) => e.state == ROOM_KEY_REQUEST_STATES.SENT).forEach((e) => {
+            expect(r).toContainEqual(e);
+        });
+    });
+
+    test("getOutgoingRoomKeyRequestByState retrieves any entry in a given state",
+    async () => {
+        const r =
+            await store.getOutgoingRoomKeyRequestByState([ROOM_KEY_REQUEST_STATES.SENT]);
+        expect(r).not.toBeNull();
+        expect(r).not.toBeUndefined();
+        expect(r.state).toEqual(ROOM_KEY_REQUEST_STATES.SENT);
+        expect(requests).toContainEqual(r);
+    });
+});
@@ -20,6 +20,8 @@ import {SECRET_STORAGE_ALGORITHM_V1_AES} from "../../../src/crypto/SecretStorage
 import {MatrixEvent} from "../../../src/models/event";
 import {TestClient} from '../../TestClient';
 import {makeTestClients} from './verification/util';
+import {encryptAES} from "../../../src/crypto/aes";
+import {resetCrossSigningKeys, createSecretStorageKey} from "./crypto-utils";

 import * as utils from "../../../src/utils";

@@ -49,6 +51,13 @@ async function makeTestClient(userInfo, options) {
    return client;
 }

+// Wrapper around pkSign to return a signed object. pkSign returns the
+// signature, rather than the signed object.
+function sign(obj, key, userId) {
+    olmlib.pkSign(obj, key, userId);
+    return obj;
+}
+
 describe("Secrets", function() {
    if (!global.Olm) {
        console.warn('Not running megolm backup unit tests: libolm not present');
@@ -182,7 +191,7 @@ describe("Secrets", function() {
                }),
            ]);
        };
-        alice.resetCrossSigningKeys();
+        resetCrossSigningKeys(alice);

        const newKeyId = await alice.addSecretStorageKey(
            SECRET_STORAGE_ALGORITHM_V1_AES,
@@ -266,100 +275,399 @@ describe("Secrets", function() {
        expect(secret).toBe("bar");
    });

-    it("bootstraps when no storage or cross-signing keys locally", async function() {
-        const key = new Uint8Array(16);
-        for (let i = 0; i < 16; i++) key[i] = i;
-        const getKey = jest.fn(e => {
-            return [Object.keys(e.keys)[0], key];
-        });
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: getKey,
-                },
-            },
+    describe("bootstrap", function() {
+        // keys used in some of the tests
+        const XSK = new Uint8Array(
+            olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="),
        );
-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+        const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
+        const USK = new Uint8Array(
+            olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="),
+        );
+        const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
+        const SSK = new Uint8Array(
+            olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="),
+        );
+        const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
+        const SSSSKey = new Uint8Array(
+            olmlib.decodeBase64(
+                "XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0=",
+            ),
+        );
+
+        it("bootstraps when no storage or cross-signing keys locally", async function() {
+            const key = new Uint8Array(16);
+            for (let i = 0; i < 16; i++) key[i] = i;
+            const getKey = jest.fn(e => {
+                return [Object.keys(e.keys)[0], key];
            });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };

-        await bob.bootstrapSecretStorage();
-
-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
-
-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
-    });
-
-    it("bootstraps when cross-signing keys in secret storage", async function() {
-        const decryption = new global.Olm.PkDecryption();
-        const storagePublicKey = decryption.generate_key();
-        const storagePrivateKey = decryption.get_private_key();
-
-        const bob = await makeTestClient(
-            {
-                userId: "@bob:example.com",
-                deviceId: "bob1",
-            },
-            {
-                cryptoCallbacks: {
-                    getSecretStorageKey: async request => {
-                        const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
-                        expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
-                        return [defaultKeyId, storagePrivateKey];
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: getKey,
                    },
                },
-            },
-        );
+            );
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };

-        bob.uploadDeviceSigningKeys = async () => {};
-        bob.uploadKeySignatures = async () => {};
-        bob.setAccountData = async function(eventType, contents, callback) {
-            const event = new MatrixEvent({
-                type: eventType,
-                content: contents,
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey,
            });
-            this.store.storeAccountDataEvents([
-                event,
-            ]);
-            this.emit("accountData", event);
-        };
-        bob._crypto.checkKeyBackup = async () => {};

-        const crossSigning = bob._crypto._crossSigningInfo;
-        const secretStorage = bob._crypto._secretStorage;
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;

-        // Set up cross-signing keys from scratch with specific storage key
-        await bob.bootstrapSecretStorage({
-            createSecretStorageKey: async () => ({ pubkey: storagePublicKey }),
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
        });

-        // Clear local cross-signing keys and read from secret storage
-        bob._crypto._deviceList.storeCrossSigningForUser(
-            "@bob:example.com",
-            crossSigning.toStorage(),
-        );
-        crossSigning.keys = {};
-        await bob.bootstrapSecretStorage();
+        it("bootstraps when cross-signing keys in secret storage", async function() {
+            const decryption = new global.Olm.PkDecryption();
+            const storagePublicKey = decryption.generate_key();
+            const storagePrivateKey = decryption.get_private_key();

-        expect(crossSigning.getId()).toBeTruthy();
-        expect(await crossSigning.isStoredInSecretStorage(secretStorage)).toBeTruthy();
-        expect(await secretStorage.hasKey()).toBeTruthy();
+            const bob = await makeTestClient(
+                {
+                    userId: "@bob:example.com",
+                    deviceId: "bob1",
+                },
+                {
+                    cryptoCallbacks: {
+                        getSecretStorageKey: async request => {
+                            const defaultKeyId = await bob.getDefaultSecretStorageKeyId();
+                            expect(Object.keys(request.keys)).toEqual([defaultKeyId]);
+                            return [defaultKeyId, storagePrivateKey];
+                        },
+                    },
+                },
+            );
+
+            bob.uploadDeviceSigningKeys = async () => {};
+            bob.uploadKeySignatures = async () => {};
+            bob.setAccountData = async function(eventType, contents, callback) {
+                const event = new MatrixEvent({
+                    type: eventType,
+                    content: contents,
+                });
+                this.store.storeAccountDataEvents([
+                    event,
+                ]);
+                this.emit("accountData", event);
+            };
+            bob._crypto.checkKeyBackup = async () => {};
+
+            const crossSigning = bob._crypto._crossSigningInfo;
+            const secretStorage = bob._crypto._secretStorage;
+
+            // Set up cross-signing keys from scratch with specific storage key
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+            await bob.bootstrapSecretStorage({
+                createSecretStorageKey: async () => ({
+                    // `pubkey` not used anymore with symmetric 4S
+                    keyInfo: { pubkey: storagePublicKey },
+                    privateKey: storagePrivateKey,
+                }),
+            });
+
+            // Clear local cross-signing keys and read from secret storage
+            bob._crypto._deviceList.storeCrossSigningForUser(
+                "@bob:example.com",
+                crossSigning.toStorage(),
+            );
+            crossSigning.keys = {};
+            await bob.bootstrapCrossSigning({
+                authUploadDeviceSigningKeys: async func => await func({}),
+            });
+
+            expect(crossSigning.getId()).toBeTruthy();
+            expect(await crossSigning.isStoredInSecretStorage(secretStorage))
+                .toBeTruthy();
+            expect(await secretStorage.hasKey()).toBeTruthy();
+        });
+
+        it("adds passphrase checking if it's lacking", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                // we never use these values, other than checking that they
+                // exist, so just use dummy values
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            expect(alice.getAccountData("m.secret_storage.default_key").getContent())
+                .toEqual({key: "key_id"});
+            const keyInfo = alice.getAccountData("m.secret_storage.key.key_id")
+                .getContent();
+            expect(keyInfo.algorithm)
+                .toEqual("m.secret_storage.v1.aes-hmac-sha2");
+            expect(keyInfo.passphrase).toEqual({
+                algorithm: "m.pbkdf2",
+                iterations: 500000,
+                salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+            });
+            expect(keyInfo).toHaveProperty("iv");
+            expect(keyInfo).toHaveProperty("mac");
+            expect(alice.checkSecretStorageKey(secretStorageKeys.key_id, keyInfo))
+                .toBeTruthy();
+        });
+        it("fixes backup keys in the wrong format", async function() {
+            let crossSigningKeys = {
+                master: XSK,
+                user_signing: USK,
+                self_signing: SSK,
+            };
+            const secretStorageKeys = {
+                key_id: SSSSKey,
+            };
+            const alice = await makeTestClient(
+                {userId: "@alice:example.com", deviceId: "Osborne2"},
+                {
+                    cryptoCallbacks: {
+                        getCrossSigningKey: t => crossSigningKeys[t],
+                        saveCrossSigningKeys: k => crossSigningKeys = k,
+                        getSecretStorageKey: ({keys}, name) => {
+                            for (const keyId of Object.keys(keys)) {
+                                if (secretStorageKeys[keyId]) {
+                                    return [keyId, secretStorageKeys[keyId]];
+                                }
+                            }
+                        },
+                    },
+                },
+            );
+            alice.store.storeAccountDataEvents([
+                new MatrixEvent({
+                    type: "m.secret_storage.default_key",
+                    content: {
+                        key: "key_id",
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.secret_storage.key.key_id",
+                    content: {
+                        algorithm: "m.secret_storage.v1.aes-hmac-sha2",
+                        passphrase: {
+                            algorithm: "m.pbkdf2",
+                            iterations: 500000,
+                            salt: "GbkvwKHVMveo1zGVSb2GMMdCinG2npJK",
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.master",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.self_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.cross_signing.user_signing",
+                    content: {
+                        encrypted: {
+                            key_id: {ciphertext: "bla", mac: "bla", iv: "bla"},
+                        },
+                    },
+                }),
+                new MatrixEvent({
+                    type: "m.megolm_backup.v1",
+                    content: {
+                        encrypted: {
+                            key_id: await encryptAES(
+                                "123,45,6,7,89,1,234,56,78,90,12,34,5,67,8,90",
+                                secretStorageKeys.key_id, "m.megolm_backup.v1",
+                            ),
+                        },
+                    },
+                }),
+            ]);
+            alice._crypto._deviceList.storeCrossSigningForUser("@alice:example.com", {
+                keys: {
+                    master: {
+                        user_id: "@alice:example.com",
+                        usage: ["master"],
+                        keys: {
+                            [`ed25519:${XSPubKey}`]: XSPubKey,
+                        },
+                    },
+                    self_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["self_signing"],
+                        keys: {
+                            [`ed25519:${SSPubKey}`]: SSPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                    user_signing: sign({
+                        user_id: "@alice:example.com",
+                        usage: ["user_signing"],
+                        keys: {
+                            [`ed25519:${USPubKey}`]: USPubKey,
+                        },
+                    }, XSK, "@alice:example.com"),
+                },
+            });
+            alice.getKeyBackupVersion = async () => {
+                return {
+                    version: "1",
+                    algorithm: "m.megolm_backup.v1.curve25519-aes-sha2",
+                    auth_data: sign({
+                        public_key: "pxEXhg+4vdMf/kFwP4bVawFWdb0EmytL3eFJx++zQ0A",
+                    }, XSK, "@alice:example.com"),
+                };
+            };
+            alice.setAccountData = async function(name, data) {
+                const event = new MatrixEvent({
+                    type: name,
+                    content: data,
+                });
+                alice.store.storeAccountDataEvents([event]);
+                this.emit("accountData", event);
+            };
+
+            await alice.bootstrapSecretStorage();
+
+            const backupKey = alice.getAccountData("m.megolm_backup.v1")
+                .getContent();
+            expect(backupKey.encrypted).toHaveProperty("key_id");
+            expect(await alice.getSecret("m.megolm_backup.v1"))
+                .toEqual("ey0GB1kB6jhOWgwiBUMIWg==");
+        });
    });
 });
@@ -22,6 +22,7 @@ import {DeviceInfo} from "../../../../src/crypto/deviceinfo";
 import {verificationMethods} from "../../../../src/crypto";
 import * as olmlib from "../../../../src/crypto/olmlib";
 import {logger} from "../../../../src/logger";
+import {resetCrossSigningKeys} from "../crypto-utils";

 const Olm = global.Olm;

@@ -44,7 +45,16 @@ describe("SAS verification", function() {
    });

    it("should error on an unexpected event", async function() {
-        const sas = new SAS({}, "@alice:example.com", "ABCDEFG");
+        //channel, baseApis, userId, deviceId, startEvent, request
+        const request = {
+            onVerifierCancelled: function() {},
+        };
+        const channel = {
+            send: function() {
+                return Promise.resolve();
+            },
+        };
+        const sas = new SAS(channel, {}, "@alice:example.com", "ABCDEFG", null, request);
        sas.handleEvent(new MatrixEvent({
            sender: "@alice:example.com",
            type: "es.inquisition",
@@ -172,11 +182,14 @@ describe("SAS verification", function() {

        it("should verify a key", async () => {
            let macMethod;
+            let keyAgreement;
            const origSendToDevice = bob.client.sendToDevice.bind(bob.client);
            bob.client.sendToDevice = function(type, map) {
                if (type === "m.key.verification.accept") {
                    macMethod = map[alice.client.getUserId()][alice.client.deviceId]
                        .message_authentication_code;
+                    keyAgreement = map[alice.client.getUserId()][alice.client.deviceId]
+                        .key_agreement_protocol;
                }
                return origSendToDevice(type, map);
            };
@@ -203,6 +216,7 @@ describe("SAS verification", function() {

            // make sure that it uses the preferred method
            expect(macMethod).toBe("hkdf-hmac-sha256");
+            expect(keyAgreement).toBe("curve25519-hkdf-sha256");

            // make sure Alice and Bob verified each other
            const bobDevice
@@ -275,12 +289,12 @@ describe("SAS verification", function() {
            );
            alice.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
            alice.httpBackend.flush(undefined, 2);
-            await alice.client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(alice.client);
            bob.httpBackend.when('POST', '/keys/device_signing/upload').respond(200, {});
            bob.httpBackend.when('POST', '/keys/signatures/upload').respond(200, {});
            bob.httpBackend.flush(undefined, 2);

-            await bob.client.resetCrossSigningKeys();
+            await resetCrossSigningKeys(bob.client);

            bob.client._crypto._deviceList.storeCrossSigningForUser(
                "@alice:example.com", {
@@ -54,6 +54,7 @@ describe("self-verifications", () => {
            cacheCallbacks,
        );
        _crossSigningInfo.keys = {
+            master: { keys: { X: testKeyPub } },
            self_signing: { keys: { X: testKeyPub } },
            user_signing: { keys: { X: testKeyPub } },
        };
@@ -64,12 +65,20 @@ describe("self-verifications", () => {
            }),
        };

+        const storeSessionBackupPrivateKey = jest.fn();
+        const restoreKeyBackupWithCache = jest.fn(() => Promise.resolve());
+
        const client = {
            _crypto: {
                _crossSigningInfo,
                _secretStorage,
+                storeSessionBackupPrivateKey,
+                getSessionBackupPrivateKey: () => null,
            },
+            requestSecret: _secretStorage.request.bind(_secretStorage),
            getUserId: () => userId,
+            getKeyBackupVersion: () => Promise.resolve({}),
+            restoreKeyBackupWithCache,
        };

        const request = {
@@ -88,15 +97,20 @@ describe("self-verifications", () => {

        const result = await verification.done();

-        /* We should request, and store, two keys */
-        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(2);
-        expect(_secretStorage.request.mock.calls.length).toBe(2);
+        /* We should request, and store, 3 cross signing keys and the key backup key */
+        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls.length).toBe(3);
+        expect(_secretStorage.request.mock.calls.length).toBe(4);

        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[0][1])
          .toEqual(testKey);
        expect(cacheCallbacks.storeCrossSigningKeyCache.mock.calls[1][1])
          .toEqual(testKey);

+        expect(storeSessionBackupPrivateKey.mock.calls[0][0])
+          .toEqual(testKey);
+
+        expect(restoreKeyBackupWithCache).toHaveBeenCalled();
+
        expect(result).toBeInstanceOf(Array);
        expect(result[0][0]).toBe(testKeyPub);
        expect(result[1][0]).toBe(testKeyPub);
@@ -119,6 +119,8 @@ async function distributeEvent(ownRequest, theirRequest, event) {
    await theirRequest.channel.handleEvent(event, theirRequest, true);
 }

+jest.useFakeTimers();
+
 describe("verification request unit tests", function() {
    beforeAll(function() {
        setupWebcrypto();
@@ -246,4 +248,38 @@ describe("verification request unit tests", function() {
        expect(bob1Request.done).toBe(true);
        expect(bob2Request.done).toBe(true);
    });
+
+    it("request times out after 10 minutes", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        await aliceRequest.channel.handleEvent(requestEvent, aliceRequest, true,
+            true, true);
+
+        expect(aliceRequest.cancelled).toBe(false);
+        expect(aliceRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(10 * 60 * 1000);
+        expect(aliceRequest._cancellingUserId).toBe(alice.getUserId());
+    });
+
+    it("request times out 2 minutes after receipt", async function() {
+        const alice = makeMockClient("@alice:matrix.tld", "device1");
+        const bob = makeMockClient("@bob:matrix.tld", "device1");
+        const aliceRequest = new VerificationRequest(
+            new InRoomChannel(alice, "!room", bob.getUserId()), new Map(), alice);
+        await aliceRequest.sendRequest();
+        const [requestEvent] = alice.popEvents();
+        const bobRequest = new VerificationRequest(
+            new InRoomChannel(bob, "!room"), new Map(), bob);
+
+        await bobRequest.channel.handleEvent(requestEvent, bobRequest, true);
+
+        expect(bobRequest.cancelled).toBe(false);
+        expect(bobRequest._cancellingUserId).toBe(undefined);
+        jest.advanceTimersByTime(2 * 60 * 1000);
+        expect(bobRequest._cancellingUserId).toBe(bob.getUserId());
+    });
 });
@@ -0,0 +1,34 @@
+import {FilterComponent} from "../../src/filter-component";
+import {mkEvent} from '../test-utils';
+
+describe("Filter Component", function() {
+    describe("types", function() {
+        it("should filter out events with other types", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.member',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(false);
+        });
+
+        it("should validate events with the same type", function() {
+            const filter = new FilterComponent({ types: ['m.room.message'] });
+            const event = mkEvent({
+                type: 'm.room.message',
+                content: { },
+                room: 'roomId',
+                event: true,
+            });
+
+            const checkResult = filter.check(event);
+
+            expect(checkResult).toBe(true);
+        });
+    });
+});
@@ -97,7 +97,7 @@ describe("InteractiveAuth", function() {
        // first we expect a call to doRequest
        doRequest.mockImplementation(function(authData) {
            logger.log("request1", authData);
-            expect(authData).toEqual({});
+            expect(authData).toEqual(null); // first request should be null
            const err = new MatrixError({
                session: "sessionId",
                flows: [
@@ -143,4 +143,33 @@ describe("InteractiveAuth", function() {
            expect(stateUpdated).toBeCalledTimes(1);
        });
    });
+
+    it("should start an auth stage and reject if no auth flow", function() {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: new FakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+        });
+
+        doRequest.mockImplementation(function(authData) {
+            logger.log("request1", authData);
+            expect(authData).toEqual(null); // first request should be null
+            const err = new MatrixError({
+                session: "sessionId",
+                flows: [],
+                params: {
+                    "logintype": { param: "aa" },
+                },
+            });
+            err.httpStatus = 401;
+            throw err;
+        });
+
+        return ia.attemptAuth().catch(function(error) {
+            expect(error.message).toBe('No appropriate authentication flow found');
+        });
+    });
 });
@@ -33,12 +33,6 @@ describe("RoomMember", function() {
            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
        });

-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.member event", function() {
-            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
-        });
-
        it("should return nothing if there is no m.room.member and allowDefault=false",
        function() {
            const url = member.getAvatarUrl(hsUrl, 64, 64, "crop", false);
@@ -45,12 +45,6 @@ describe("Room", function() {
            expect(url.indexOf("flibble/wibble")).not.toEqual(-1);
        });

-        it("should return an identicon HTTP URL if allowDefault was set and there " +
-        "was no m.room.avatar event", function() {
-            const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", true);
-            expect(url.indexOf("http")).toEqual(0); // don't care about form
-        });
-
        it("should return nothing if there is no m.room.avatar and allowDefault=false",
        function() {
            const url = room.getAvatarUrl(hsUrl, 64, 64, "crop", false);
@@ -1379,7 +1373,7 @@ describe("Room", function() {
            let hasThrown = false;
            try {
                await room.loadMembersIfNeeded();
-            } catch(err) {
+            } catch (err) {
                hasThrown = true;
            }
            expect(hasThrown).toEqual(true);
@@ -143,7 +143,7 @@ describe("MatrixScheduler", function() {
        deferA.reject({});
        try {
            await globalA;
-        } catch(err) {
+        } catch (err) {
            await Promise.resolve();
            expect(procCount).toEqual(2);
        }
@@ -0,0 +1,25 @@
+/*
+Copyright 2020 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+export {};
+
+declare global {
+    namespace NodeJS {
+        interface Global {
+            localStorage: Storage;
+        }
+    }
+}
@@ -502,10 +502,12 @@ export class AutoDiscovery {
            request(
                { method: "GET", uri: url, timeout: 5000 },
                (err, response, body) => {
-                    if (err || response.statusCode < 200 || response.statusCode >= 300) {
+                    if (err || response &&
+                        (response.statusCode < 200 || response.statusCode >= 300)
+                    ) {
                        let action = "FAIL_PROMPT";
                        let reason = (err ? err.message : null) || "General failure";
-                        if (response.statusCode === 404) {
+                        if (response && response.statusCode === 404) {
                            action = "IGNORE";
                            reason = AutoDiscovery.ERROR_MISSING_WELLKNOWN;
                        }
@@ -66,7 +66,7 @@ function termsUrlForService(serviceType, baseUrl) {
 * callback that returns a Promise<String> of an identity access token to supply
 * with identity requests. If the object is unset, no access token will be
 * supplied.
- * See also https://github.com/vector-im/riot-web/issues/10615 which seeks to
+ * See also https://github.com/vector-im/element-web/issues/10615 which seeks to
 * replace the previous approach of manual access tokens params with this
 * callback throughout the SDK.
 *
@@ -206,9 +206,6 @@ MatrixBaseApis.prototype.register = function(
        inhibitLogin = undefined;
    }

-    if (auth === undefined || auth === null) {
-        auth = {};
-    }
    if (sessionId) {
        auth.session = sessionId;
    }
@@ -1757,7 +1754,7 @@ MatrixBaseApis.prototype.downloadKeysForUsers = function(userIds, opts) {
        content.token = opts.token;
    }
    userIds.forEach((u) => {
-        content.device_keys[u] = {};
+        content.device_keys[u] = [];
    });

    return this._http.authedRequest(undefined, "POST", "/keys/query", undefined, content);
@@ -34,7 +34,7 @@ matrixcs.request(function(opts, fn) {
 let indexedDB;
 try {
    indexedDB = global.indexedDB;
-} catch(e) {}
+} catch (e) {}

 // if our browser (appears to) support indexeddb, use an indexeddb crypto store.
 if (indexedDB) {
@@ -35,19 +35,26 @@ import {StubStore} from "./store/stub";
 import {createNewMatrixCall} from "./webrtc/call";
 import * as utils from './utils';
 import {sleep} from './utils';
-import {MatrixError, PREFIX_MEDIA_R0, PREFIX_UNSTABLE} from "./http-api";
+import {
+    MatrixError,
+    PREFIX_MEDIA_R0,
+    PREFIX_UNSTABLE,
+    retryNetworkOperation,
+} from "./http-api";
 import {getHttpUriForMxc} from "./content-repo";
 import * as ContentHelpers from "./content-helpers";
 import * as olmlib from "./crypto/olmlib";
 import {ReEmitter} from './ReEmitter';
 import {RoomList} from './crypto/RoomList';
 import {logger} from './logger';
-import {Crypto, isCryptoAvailable} from './crypto';
+import {Crypto, isCryptoAvailable, fixBackupKey} from './crypto';
 import {decodeRecoveryKey} from './crypto/recoverykey';
 import {keyFromAuthData} from './crypto/key_passphrase';
 import {randomString} from './randomstring';
 import {PushProcessor} from "./pushprocessor";
 import {encodeBase64, decodeBase64} from "./crypto/olmlib";
+import { User } from "./models/user";
+import {AutoDiscovery} from "./autodiscovery";

 const SCROLLBACK_DELAY_MS = 3000;
 export const CRYPTO_ENABLED = isCryptoAvailable();
@@ -104,12 +111,15 @@ function keyFromRecoverySession(session, decryptionKey) {
 *     If provided, opts.deviceId and opts.userId should **NOT** be provided
 *     (they are present in the exported data).
 *
+ * @param {string} opts.pickleKey Key used to pickle olm objects or other
+ *     sensitive data.
+ *
 * @param {IdentityServerProvider} [opts.identityServer]
 * Optional. A provider object with one function `getAccessToken`, which is a
 * callback that returns a Promise<String> of an identity access token to supply
 * with identity requests. If the object is unset, no access token will be
 * supplied.
- * See also https://github.com/vector-im/riot-web/issues/10615 which seeks to
+ * See also https://github.com/vector-im/element-web/issues/10615 which seeks to
 * replace the previous approach of manual access tokens params with this
 * callback throughout the SDK.
 *
@@ -225,6 +235,14 @@ function keyFromRecoverySession(session, decryptionKey) {
 *       }
 *   {string} name the name of the value we want to read out of SSSS, for UI purposes.
 *
+ * @param {function} [opts.cryptoCallbacks.cacheSecretStorageKey]
+ * Optional. Function called when a new encryption key for secret storage
+ * has been created. This allows the application a chance to cache this key if
+ * desired to avoid user prompts.
+ * Args:
+ *   {string} keyId the ID of the new key
+ *   {Uint8Array} key the new private key
+ *
 * @param {function} [opts.cryptoCallbacks.onSecretRequested]
 * Optional. Function called when a request for a secret is received from another
 * device.
@@ -278,6 +296,8 @@ export function MatrixClient(opts) {
            // will be used during async initialization of the crypto
            this._exportedOlmDeviceToImport = opts.deviceToImport.olmDevice;
        }
+    } else if (opts.pickleKey) {
+        this.pickleKey = opts.pickleKey;
    }

    this.scheduler = opts.scheduler;
@@ -318,7 +338,7 @@ export function MatrixClient(opts) {
    this._isGuest = false;
    this._ongoingScrollbacks = {};
    this.timelineSupport = Boolean(opts.timelineSupport);
-    this.urlPreviewCache = {};
+    this.urlPreviewCache = {}; // key=preview key, value=Promise for preview (may be an error)
    this._notifTimelineSet = null;
    this.unstableClientRelationAggregation = !!opts.unstableClientRelationAggregation;

@@ -345,6 +365,9 @@ export function MatrixClient(opts) {

    this._cachedCapabilities = null; // { capabilities: {}, lastUpdated: timestamp }

+    this._clientWellKnown = undefined;
+    this._clientWellKnownPromise = undefined;
+
    // The SDK doesn't really provide a clean way for events to recalculate the push
    // actions for themselves, so we have to kinda help them out when they are encrypted.
    // We do this so that push rules are correctly executed on events in their decrypted
@@ -368,7 +391,7 @@ export function MatrixClient(opts) {
            ? !!actions.tweaks.highlight : false;
        if (oldHighlight !== newHighlight || currentCount > 0) {
            // TODO: Handle mentions received while the client is offline
-            // See also https://github.com/vector-im/riot-web/issues/9069
+            // See also https://github.com/vector-im/element-web/issues/9069
            if (!room.hasUserReadEvent(this.getUserId(), event.getId())) {
                let newCount = currentCount;
                if (newHighlight && !oldHighlight) newCount++;
@@ -386,7 +409,7 @@ export function MatrixClient(opts) {

    // Like above, we have to listen for read receipts from ourselves in order to
    // correctly handle notification counts on encrypted rooms.
-    // This fixes https://github.com/vector-im/riot-web/issues/9421
+    // This fixes https://github.com/vector-im/element-web/issues/9421
    this.on("Room.receipt", (event, room) => {
        if (room && this.isRoomEncrypted(room.roomId)) {
            // Figure out if we've read something or if it's just informational
@@ -731,6 +754,7 @@ MatrixClient.prototype.initCrypto = async function() {
        "crypto.roomKeyRequestCancellation",
        "crypto.warning",
        "crypto.devicesUpdated",
+        "crypto.willUpdateDevices",
        "deviceVerificationChanged",
        "userTrustStatusChanged",
        "crossSigning.keysChanged",
@@ -739,6 +763,7 @@ MatrixClient.prototype.initCrypto = async function() {
    logger.log("Crypto: initialising crypto object...");
    await crypto.init({
        exportedOlmDevice: this._exportedOlmDeviceToImport,
+        pickleKey: this.pickleKey,
    });
    delete this._exportedOlmDeviceToImport;

@@ -819,9 +844,9 @@ MatrixClient.prototype.downloadKeys = function(userIds, forceDownload) {
 *
 * @param {string} userId the user to list keys for.
 *
- * @return {Promise<module:crypto/deviceinfo[]>} list of devices
+ * @return {module:crypto/deviceinfo[]} list of devices
 */
-MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
+MatrixClient.prototype.getStoredDevicesForUser = function(userId) {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
    }
@@ -834,9 +859,9 @@ MatrixClient.prototype.getStoredDevicesForUser = async function(userId) {
 * @param {string} userId the user to list keys for.
 * @param {string} deviceId unique identifier for the device
 *
- * @return {Promise<?module:crypto/deviceinfo>} device or null
+ * @return {module:crypto/deviceinfo} device or null
 */
-MatrixClient.prototype.getStoredDevice = async function(userId, deviceId) {
+MatrixClient.prototype.getStoredDevice = function(userId, deviceId) {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
    }
@@ -955,6 +980,20 @@ MatrixClient.prototype.findVerificationRequestDMInProgress = function(roomId) {
    return this._crypto.findVerificationRequestDMInProgress(roomId);
 };

+/**
+ * Returns all to-device verification requests that are already in progress for the given user id
+ *
+ * @param {string} userId the ID of the user to query
+ *
+ * @returns {module:crypto/verification/request/VerificationRequest[]} the VerificationRequests that are in progress
+ */
+MatrixClient.prototype.getVerificationRequestsToDeviceInProgress = function(userId) {
+    if (this._crypto === null) {
+        throw new Error("End-to-end encryption disabled");
+    }
+    return this._crypto.getVerificationRequestsToDeviceInProgress(userId);
+};
+
 /**
 * Request a key verification from another user.
 *
@@ -1061,19 +1100,9 @@ function wrapCryptoFuncs(MatrixClient, names) {
    }
 }

- /**
- * Generate new cross-signing keys.
- * The cross-signing API is currently UNSTABLE and may change without notice.
- *
- * @function module:client~MatrixClient#resetCrossSigningKeys
- * @param {object} authDict Auth data to supply for User-Interactive auth.
- * @param {CrossSigningLevel} [level] the level of cross-signing to reset.  New
- * keys will be created for the given level and below.  Defaults to
- * regenerating all keys.
- */
-
 /**
 * Get the user's cross-signing key ID.
+ *
 * The cross-signing API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#getCrossSigningId
@@ -1085,6 +1114,7 @@ function wrapCryptoFuncs(MatrixClient, names) {

 /**
 * Get the cross signing information for a given user.
+ *
 * The cross-signing API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#getStoredCrossSigningForUser
@@ -1095,6 +1125,7 @@ function wrapCryptoFuncs(MatrixClient, names) {

 /**
 * Check whether a given user is trusted.
+ *
 * The cross-signing API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#checkUserTrust
@@ -1105,6 +1136,7 @@ function wrapCryptoFuncs(MatrixClient, names) {

 /**
 * Check whether a given device is trusted.
+ *
 * The cross-signing API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#checkDeviceTrust
@@ -1117,6 +1149,7 @@ function wrapCryptoFuncs(MatrixClient, names) {
 /**
 * Check the copy of our cross-signing key that we have in the device list and
 * see if we can get the private key. If so, mark it as trusted.
+ *
 * The cross-signing API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#checkOwnCrossSigningTrust
@@ -1126,6 +1159,7 @@ function wrapCryptoFuncs(MatrixClient, names) {
 * Checks that a given cross-signing private key matches a given public key.
 * This can be used by the getCrossSigningKey callback to verify that the
 * private key it is about to supply is the one that was requested.
+ *
 * The cross-signing API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#checkCrossSigningPrivateKey
@@ -1138,10 +1172,49 @@ function wrapCryptoFuncs(MatrixClient, names) {
 * Perform any background tasks that can be done before a message is ready to
 * send, in order to speed up sending of the message.
 *
+ * @function module:client~MatrixClient#prepareToEncrypt
 * @param {module:models/room} room the room the event is in
 */
+
+/**
+ * Checks whether cross signing:
+ * - is enabled on this account and trusted by this device
+ * - has private keys either cached locally or stored in secret storage
+ *
+ * If this function returns false, bootstrapCrossSigning() can be used
+ * to fix things such that it returns true. That is to say, after
+ * bootstrapCrossSigning() completes successfully, this function should
+ * return true.
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @function module:client~MatrixClient#isCrossSigningReady
+ * @return {bool} True if cross-signing is ready to be used on this device
+ */
+
+/**
+ * Bootstrap cross-signing by creating keys if needed. If everything is already
+ * set up, then no changes are made, so this is safe to run to ensure
+ * cross-signing is ready for use.
+ *
+ * This function:
+ * - creates new cross-signing keys if they are not found locally cached nor in
+ *   secret storage (if it has been setup)
+ *
+ * The cross-signing API is currently UNSTABLE and may change without notice.
+ *
+ * @function module:client~MatrixClient#bootstrapCrossSigning
+ * @param {function} opts.authUploadDeviceSigningKeys Function
+ * called to await an interactive auth flow when uploading device signing keys.
+ * @param {bool} [opts.setupNewCrossSigning] Optional. Reset even if keys
+ * already exist.
+ * Args:
+ *     {function} A function that makes the request requiring auth. Receives the
+ *     auth data as an object. Can be called multiple times, first with an empty
+ *     authDict, to obtain the flows.
+ */
+
 wrapCryptoFuncs(MatrixClient, [
-    "resetCrossSigningKeys",
    "getCrossSigningId",
    "getStoredCrossSigningForUser",
    "checkUserTrust",
@@ -1151,54 +1224,99 @@ wrapCryptoFuncs(MatrixClient, [
    "legacyDeviceVerification",
    "prepareToEncrypt",
    "isCrossSigningReady",
+    "bootstrapCrossSigning",
    "getCryptoTrustCrossSignedDevices",
    "setCryptoTrustCrossSignedDevices",
+    "countSessionsNeedingBackup",
 ]);

 /**
- * Check if the sender of an event is verified
- * The cross-signing API is currently UNSTABLE and may change without notice.
+ * Get information about the encryption of an event
 *
- * @param {MatrixEvent} event event to be checked
+ * @function module:client~MatrixClient#getEventEncryptionInfo
 *
- * @returns {DeviceTrustLevel}
+ * @param {module:models/event.MatrixEvent} event event to be checked
+ *
+ * @return {object} An object with the fields:
+ *    - encrypted: whether the event is encrypted (if not encrypted, some of the
+ *      other properties may not be set)
+ *    - senderKey: the sender's key
+ *    - algorithm: the algorithm used to encrypt the event
+ *    - authenticated: whether we can be sure that the owner of the senderKey
+ *      sent the event
+ *    - sender: the sender's device information, if available
+ *    - mismatchedSender: if the event's ed25519 and curve25519 keys don't match
+ *      (only meaningful if `sender` is set)
 */
-MatrixClient.prototype.checkEventSenderTrust = async function(event) {
-    const device = await this.getEventSenderDeviceInfo(event);
-    if (!device) {
-        return 0;
-    }
-    return await this._crypto.checkDeviceTrust(event.getSender(), device.deviceId);
-};

 /**
 * Create a recovery key from a user-supplied passphrase.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#createRecoveryKeyFromPassphrase
 * @param {string} password Passphrase string that can be entered by the user
 *     when restoring the backup as an alternative to entering the recovery key.
 *     Optional.
- * @returns {Promise<String>} The user-facing recovery key string.
+ * @returns {Promise<Object>} Object with public key metadata, encoded private
+ *     recovery key which should be disposed of after displaying to the user,
+ *     and raw private key to avoid round tripping if needed.
 */

 /**
- * Bootstrap Secure Secret Storage if needed by creating a default key and signing it with
- * the cross-signing master key. If everything is already set up, then no
- * changes are made, so this is safe to run to ensure secret storage is ready
- * for use.
+ * Checks whether secret storage:
+ * - is enabled on this account
+ * - is storing cross-signing private keys
+ * - is storing session backup key (if enabled)
+ *
+ * If this function returns false, bootstrapSecretStorage() can be used
+ * to fix things such that it returns true. That is to say, after
+ * bootstrapSecretStorage() completes successfully, this function should
+ * return true.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
+ * @function module:client~MatrixClient#isSecretStorageReady
+ * @return {bool} True if secret storage is ready to be used on this device
+ */
+
+/**
+ * Bootstrap Secure Secret Storage if needed by creating a default key. If everything is
+ * already set up, then no changes are made, so this is safe to run to ensure secret
+ * storage is ready for use.
+ *
+ * This function
+ * - creates a new Secure Secret Storage key if no default key exists
+ *   - if a key backup exists, it is migrated to store the key in the Secret
+ *     Storage
+ * - creates a backup if none exists, and one is requested
+ * - migrates Secure Secret Storage to use the latest algorithm, if an outdated
+ *   algorithm is found
+ *
 * @function module:client~MatrixClient#bootstrapSecretStorage
- * @param {function} [opts.authUploadDeviceSigningKeys] Optional. Function
- * called to await an interactive auth flow when uploading device signing keys.
- * Args:
- *     {function} A function that makes the request requiring auth. Receives the
- *     auth data as an object.
+ * @param {function} [opts.createSecretStorageKey] Optional. Function
+ * called to await a secret storage key creation flow.
+ * Returns:
+ *     {Promise<Object>} Object with public key metadata, encoded private
+ *     recovery key which should be disposed of after displaying to the user,
+ *     and raw private key to avoid round tripping if needed.
+ * @param {object} [opts.keyBackupInfo] The current key backup object. If passed,
+ * the passphrase and recovery key from this backup will be used.
+ * @param {bool} [opts.setupNewKeyBackup] If true, a new key backup version will be
+ * created and the private key stored in the new SSSS store. Ignored if keyBackupInfo
+ * is supplied.
+ * @param {bool} [opts.setupNewSecretStorage] Optional. Reset even if keys already exist.
+ * @param {func} [opts.getKeyBackupPassphrase] Optional. Function called to get the user's
+ *     current key backup passphrase. Should return a promise that resolves with a Buffer
+ *     containing the key, or rejects if the key cannot be obtained.
+ * Returns:
+ *     {Promise} A promise which resolves to key creation data for
+ *     SecretStorage#addKey: an object with `passphrase` and/or `pubkey` fields.
 */

 /**
 * Add a key for encrypting secrets.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#addSecretStorageKey
@@ -1213,6 +1331,7 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {

 /**
 * Check whether we have a key with a given ID.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#hasSecretStorageKey
@@ -1222,7 +1341,8 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {
 */

 /**
- * Store an encrypted secret on the server
+ * Store an encrypted secret on the server.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#storeSecret
@@ -1234,6 +1354,7 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {

 /**
 * Get a secret from storage.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#getSecret
@@ -1244,6 +1365,7 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {

 /**
 * Check if a secret is stored on the server.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#isSecretStored
@@ -1258,6 +1380,7 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {

 /**
 * Request a secret from another device.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#requestSecret
@@ -1269,6 +1392,7 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {

 /**
 * Get the current default key ID for encrypting secrets.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#getDefaultSecretStorageKeyId
@@ -1278,6 +1402,7 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {

 /**
 * Set the current default key ID for encrypting secrets.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#setDefaultSecretStorageKeyId
@@ -1288,6 +1413,7 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {
 * Checks that a given secret storage private key matches a given public key.
 * This can be used by the getSecretStorageKey callback to verify that the
 * private key it is about to supply is the one that was requested.
+ *
 * The Secure Secret Storage API is currently UNSTABLE and may change without notice.
 *
 * @function module:client~MatrixClient#checkSecretStoragePrivateKey
@@ -1297,17 +1423,19 @@ MatrixClient.prototype.checkEventSenderTrust = async function(event) {
 */

 wrapCryptoFuncs(MatrixClient, [
+    "getEventEncryptionInfo",
    "createRecoveryKeyFromPassphrase",
+    "isSecretStorageReady",
    "bootstrapSecretStorage",
    "addSecretStorageKey",
    "hasSecretStorageKey",
-    "secretStorageKeyNeedsUpgrade",
    "storeSecret",
    "getSecret",
    "isSecretStored",
    "requestSecret",
    "getDefaultSecretStorageKeyId",
    "setDefaultSecretStorageKeyId",
+    "checkSecretStorageKey",
    "checkSecretStoragePrivateKey",
 ]);

@@ -1354,7 +1482,8 @@ MatrixClient.prototype.cancelAndResendEventRoomKeyRequest = function(event) {
 };

 /**
- * Enable end-to-end encryption for a room.
+ * Enable end-to-end encryption for a room. This does not modify room state.
+ * Any messages sent before the returned promise resolves will be sent unencrypted.
 * @param {string} roomId The room ID to enable encryption in.
 * @param {object} config The encryption config for the room.
 * @return {Promise} A promise that will resolve when encryption is set up.
@@ -1426,15 +1555,17 @@ MatrixClient.prototype.exportRoomKeys = function() {
 * Import a list of room keys previously exported by exportRoomKeys
 *
 * @param {Object[]} keys a list of session export objects
+ * @param {Object} opts
+ * @param {Function} opts.progressCallback called with an object that has a "stage" param
 *
 * @return {Promise} a promise which resolves when the keys
 *    have been imported
 */
-MatrixClient.prototype.importRoomKeys = function(keys) {
+MatrixClient.prototype.importRoomKeys = function(keys, opts) {
    if (!this._crypto) {
        throw new Error("End-to-end encryption disabled");
    }
-    return this._crypto.importRoomKeys(keys);
+    return this._crypto.importRoomKeys(keys, opts);
 };

 /**
@@ -1494,12 +1625,16 @@ MatrixClient.prototype.isKeyBackupTrusted = function(info) {

 /**
 * @returns {bool} true if the client is configured to back up keys to
- *     the server, otherwise false.
+ *     the server, otherwise false. If we haven't completed a successful check
+ *     of key backup status yet, returns null.
 */
 MatrixClient.prototype.getKeyBackupEnabled = function() {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
    }
+    if (!this._crypto._checkedForBackup) {
+        return null;
+    }
    return Boolean(this._crypto.backupKey);
 };

@@ -1563,7 +1698,7 @@ MatrixClient.prototype.prepareKeyBackupVersion = async function(
        throw new Error("End-to-end encryption disabled");
    }

-    const [keyInfo, encodedPrivateKey, privateKey] =
+    const { keyInfo, encodedPrivateKey, privateKey } =
        await this.createRecoveryKeyFromPassphrase(password);

    if (secureSecretStorage) {
@@ -1812,7 +1947,17 @@ MatrixClient.prototype.restoreKeyBackupWithPassword = async function(
 MatrixClient.prototype.restoreKeyBackupWithSecretStorage = async function(
    backupInfo, targetRoomId, targetSessionId, opts,
 ) {
-    const privKey = decodeBase64(await this.getSecret("m.megolm_backup.v1"));
+    const storedKey = await this.getSecret("m.megolm_backup.v1");
+
+    // ensure that the key is in the right format.  If not, fix the key and
+    // store the fixed version
+    const fixedKey = fixBackupKey(storedKey);
+    if (fixedKey) {
+        const [keyId] = await this._crypto.getSecretStorageKey();
+        await this.storeSecret("m.megolm_backup.v1", fixedKey, [keyId]);
+    }
+
+    const privKey = decodeBase64(fixedKey || storedKey);
    return this._restoreKeyBackup(
        privKey, targetRoomId, targetSessionId, backupInfo, opts,
    );
@@ -1867,7 +2012,10 @@ MatrixClient.prototype.restoreKeyBackupWithCache = async function(

 MatrixClient.prototype._restoreKeyBackup = function(
    privKey, targetRoomId, targetSessionId, backupInfo,
-    { cacheCompleteCallback }={}, // For sequencing during tests
+    {
+        cacheCompleteCallback, // For sequencing during tests
+        progressCallback,
+    }={},
 ) {
    if (this._crypto === null) {
        throw new Error("End-to-end encryption disabled");
@@ -1883,7 +2031,7 @@ MatrixClient.prototype._restoreKeyBackup = function(
    let backupPubKey;
    try {
        backupPubKey = decryption.init_with_private_key(privKey);
-    } catch(e) {
+    } catch (e) {
        decryption.free();
        throw e;
    }
@@ -1902,6 +2050,12 @@ MatrixClient.prototype._restoreKeyBackup = function(
        console.warn("Error caching session backup key:", e);
    }).then(cacheCompleteCallback);

+    if (progressCallback) {
+        progressCallback({
+            stage: "fetch",
+        });
+    }
+
    return this._http.authedRequest(
        undefined, "GET", path.path, path.queryData, undefined,
        {prefix: PREFIX_UNSTABLE},
@@ -1936,7 +2090,11 @@ MatrixClient.prototype._restoreKeyBackup = function(
            }
        }

-        return this.importRoomKeys(keys);
+        return this.importRoomKeys(keys, {
+            progressCallback,
+            untrusted: true,
+            source: "backup",
+        });
    }).then(() => {
        return this._crypto.setTrustedBackupPubKey(backupPubKey);
    }).then(() => {
@@ -2072,6 +2230,7 @@ MatrixClient.prototype.getUsers = function() {

 /**
 * Set account data event for the current user.
+ * It will retry the request up to 5 times.
 * @param {string} eventType The event type
 * @param {Object} contents the contents object for the event
 * @param {module:client.callback} callback Optional.
@@ -2083,9 +2242,13 @@ MatrixClient.prototype.setAccountData = function(eventType, contents, callback)
        $userId: this.credentials.userId,
        $type: eventType,
    });
-    return this._http.authedRequest(
-        callback, "PUT", path, undefined, contents,
-    );
+    const promise = retryNetworkOperation(5, () => {
+        return this._http.authedRequest(undefined, "PUT", path, undefined, contents);
+    });
+    if (callback) {
+        promise.then(result => callback(null, result), callback);
+    }
+    return promise;
 };

 /**
@@ -2120,9 +2283,17 @@ MatrixClient.prototype.getAccountDataFromServer = async function(eventType) {
        $userId: this.credentials.userId,
        $type: eventType,
    });
-    return this._http.authedRequest(
-        undefined, "GET", path, undefined,
-    );
+    try {
+        const result = await this._http.authedRequest(
+            undefined, "GET", path, undefined,
+        );
+        return result;
+    } catch (e) {
+        if (e.data && e.data.errcode === 'M_NOT_FOUND') {
+            return null;
+        }
+        throw e;
+    }
 };

 /**
@@ -2436,6 +2607,7 @@ MatrixClient.prototype._sendCompleteEvent = function(roomId, eventObject, txnId,
    const localEvent = new MatrixEvent(Object.assign(eventObject, {
        event_id: "~" + roomId + ":" + txnId,
        user_id: this.credentials.userId,
+        sender: this.credentials.userId,
        room_id: roomId,
        origin_server_ts: new Date().getTime(),
    }));
@@ -2457,7 +2629,7 @@ MatrixClient.prototype._sendCompleteEvent = function(roomId, eventObject, txnId,
    const type = localEvent.getType();
    logger.log(`sendEvent of type ${type} in ${roomId} with txnId ${txnId}`);

-    localEvent._txnId = txnId;
+    localEvent.setTxnId(txnId);
    localEvent.setStatus(EventStatus.SENDING);

    // add this event immediately to the local store as 'sending'.
@@ -2625,7 +2797,11 @@ function _updatePendingEventStatus(room, event, newStatus) {
 }

 function _sendEventHttpRequest(client, event) {
-    const txnId = event._txnId ? event._txnId : client.makeTxnId();
+    let txnId = event.getTxnId();
+    if (!txnId) {
+        txnId = client.makeTxnId();
+        event.setTxnId(txnId);
+    }

    const pathParams = {
        $roomId: event.getRoomId(),
@@ -2946,25 +3122,32 @@ MatrixClient.prototype.setRoomReadMarkers = async function(
 * May return synthesized attributes if the URL lacked OG meta.
 */
 MatrixClient.prototype.getUrlPreview = function(url, ts, callback) {
+    // bucket the timestamp to the nearest minute to prevent excessive spam to the server
+    // Surely 60-second accuracy is enough for anyone.
+    ts = Math.floor(ts / 60000) * 60000;
+
    const key = ts + "_" + url;
-    const og = this.urlPreviewCache[key];
-    if (og) {
-        return Promise.resolve(og);
+
+    // If there's already a request in flight (or we've handled it), return that instead.
+    const cachedPreview = this.urlPreviewCache[key];
+    if (cachedPreview) {
+        if (callback) {
+            cachedPreview.then(callback).catch(callback);
+        }
+        return cachedPreview;
    }

-    const self = this;
-    return this._http.authedRequest(
+    const resp = this._http.authedRequest(
        callback, "GET", "/preview_url", {
            url: url,
            ts: ts,
        }, undefined, {
            prefix: PREFIX_MEDIA_R0,
        },
-    ).then(function(response) {
-        // TODO: expire cache occasionally
-        self.urlPreviewCache[key] = response;
-        return response;
-    });
+    );
+    // TODO: Expire the URL preview cache sometimes
+    this.urlPreviewCache[key] = resp;
+    return resp;
 };

 /**
@@ -3466,46 +3649,6 @@ MatrixClient.prototype.setPresence = function(opts, callback) {
    );
 };

-function _presenceList(callback, client, opts, method) {
-  const path = utils.encodeUri("/presence/list/$userId", {
-      $userId: client.credentials.userId,
-  });
-  return client._http.authedRequest(callback, method, path, undefined, opts);
-}
-
-/**
-* Retrieve current user presence list.
-* @param {module:client.callback} callback Optional.
-* @return {Promise} Resolves: TODO
-* @return {module:http-api.MatrixError} Rejects: with an error response.
-*/
-MatrixClient.prototype.getPresenceList = function(callback) {
-  return _presenceList(callback, this, undefined, "GET");
-};
-
-/**
-* Add users to the current user presence list.
-* @param {module:client.callback} callback Optional.
-* @param {string[]} userIds
-* @return {Promise} Resolves: TODO
-* @return {module:http-api.MatrixError} Rejects: with an error response.
-*/
-MatrixClient.prototype.inviteToPresenceList = function(callback, userIds) {
-  const opts = {"invite": userIds};
-  return _presenceList(callback, this, opts, "POST");
-};
-
-/**
-* Drop users from the current user presence list.
-* @param {module:client.callback} callback Optional.
-* @param {string[]} userIds
-* @return {Promise} Resolves: TODO
-* @return {module:http-api.MatrixError} Rejects: with an error response.
-**/
-MatrixClient.prototype.dropFromPresenceList = function(callback, userIds) {
-  const opts = {"drop": userIds};
-  return _presenceList(callback, this, opts, "POST");
-};

 /**
 * Retrieve older messages from the given room and put them in the timeline.
@@ -3764,7 +3907,9 @@ MatrixClient.prototype.paginateEventTimeline = function(eventTimeline, opts) {
        return pendingRequest;
    }

-    let path, params, promise;
+    let path;
+    let params;
+    let promise;
    const self = this;

    if (isNotifTimeline) {
@@ -4184,7 +4329,8 @@ MatrixClient.prototype.getRoomPushRule = function(scope, roomId) {
 */
 MatrixClient.prototype.setRoomMutePushRule = function(scope, roomId, mute) {
    const self = this;
-    let deferred, hasDontNotifyRule;
+    let deferred;
+    let hasDontNotifyRule;

    // Get the existing room-kind push rule if any
    const roomPushRule = this.getRoomPushRule(scope, roomId);
@@ -4496,64 +4642,54 @@ MatrixClient.prototype.getFilter = function(userId, filterId, allowCached) {
 * @param {Filter} filter
 * @return {Promise<String>} Filter ID
 */
-MatrixClient.prototype.getOrCreateFilter = function(filterName, filter) {
+MatrixClient.prototype.getOrCreateFilter = async function(filterName, filter) {
    const filterId = this.store.getFilterIdByName(filterName);
-    let promise = Promise.resolve();
-    const self = this;
+    let existingId = undefined;

    if (filterId) {
        // check that the existing filter matches our expectations
-        promise = self.getFilter(self.credentials.userId,
-                         filterId, true,
-        ).then(function(existingFilter) {
-            const oldDef = existingFilter.getDefinition();
-            const newDef = filter.getDefinition();
+        try {
+            const existingFilter =
+                await this.getFilter(this.credentials.userId, filterId, true);
+            if (existingFilter) {
+                const oldDef = existingFilter.getDefinition();
+                const newDef = filter.getDefinition();

-            if (utils.deepCompare(oldDef, newDef)) {
-                // super, just use that.
-                // debuglog("Using existing filter ID %s: %s", filterId,
-                //          JSON.stringify(oldDef));
-                return Promise.resolve(filterId);
+                if (utils.deepCompare(oldDef, newDef)) {
+                    // super, just use that.
+                    // debuglog("Using existing filter ID %s: %s", filterId,
+                    //          JSON.stringify(oldDef));
+                    existingId = filterId;
+                }
            }
-            // debuglog("Existing filter ID %s: %s; new filter: %s",
-            //          filterId, JSON.stringify(oldDef), JSON.stringify(newDef));
-            self.store.setFilterIdByName(filterName, undefined);
-            return undefined;
-        }, function(error) {
+        } catch (error) {
            // Synapse currently returns the following when the filter cannot be found:
            // {
            //     errcode: "M_UNKNOWN",
            //     name: "M_UNKNOWN",
            //     message: "No row found",
-            //     data: Object, httpStatus: 404
            // }
-            if (error.httpStatus === 404 &&
-                (error.errcode === "M_UNKNOWN" || error.errcode === "M_NOT_FOUND")) {
-                // Clear existing filterId from localStorage
-                // if it no longer exists on the server
-                self.store.setFilterIdByName(filterName, undefined);
-                // Return a undefined value for existingId further down the promise chain
-                return undefined;
-            } else {
+            if (error.errcode !== "M_UNKNOWN" && error.errcode !== "M_NOT_FOUND") {
                throw error;
            }
-        });
+        }
+        // if the filter doesn't exist anymore on the server, remove from store
+        if (!existingId) {
+            this.store.setFilterIdByName(filterName, undefined);
+        }
    }

-    return promise.then(function(existingId) {
-        if (existingId) {
-            return existingId;
-        }
+    if (existingId) {
+        return existingId;
+    }

-        // create a new filter
-        return self.createFilter(filter.getDefinition(),
-        ).then(function(createdFilter) {
-            // debuglog("Created new filter ID %s: %s", createdFilter.filterId,
-            //          JSON.stringify(createdFilter.getDefinition()));
-            self.store.setFilterIdByName(filterName, createdFilter.filterId);
-            return createdFilter.filterId;
-        });
-    });
+    // create a new filter
+    const createdFilter = await this.createFilter(filter.getDefinition());
+
+    // debuglog("Created new filter ID %s: %s", createdFilter.filterId,
+    //          JSON.stringify(createdFilter.getDefinition()));
+    this.store.setFilterIdByName(filterName, createdFilter.filterId);
+    return createdFilter.filterId;
 };


@@ -4717,6 +4853,9 @@ MatrixClient.prototype.deactivateSynapseUser = function(userId) {
 * @param {Boolean=} opts.lazyLoadMembers True to not load all membership events during
 * initial sync but fetch them when needed by calling `loadOutOfBandMembers`
 * This will override the filter option at this moment.
+ * @param {Number=} opts.clientWellKnownPollPeriod The number of seconds between polls
+ * to /.well-known/matrix/client, undefined to disable. This should be in the order of hours.
+ * Default: undefined.
 */
 MatrixClient.prototype.startClient = async function(opts) {
    if (this.clientRunning) {
@@ -4731,6 +4870,13 @@ MatrixClient.prototype.startClient = async function(opts) {
        };
    }

+    // Create our own user object artificially (instead of waiting for sync)
+    // so it's always available, even if the user is not in any rooms etc.
+    const userId = this.getUserId();
+    if (userId) {
+        this.store.storeUser(new User(userId));
+    }
+
    if (this._crypto) {
        this._crypto.uploadDeviceKeys();
        this._crypto.start();
@@ -4758,6 +4904,32 @@ MatrixClient.prototype.startClient = async function(opts) {
    this._clientOpts = opts;
    this._syncApi = new SyncApi(this, opts);
    this._syncApi.sync();
+
+    if (opts.clientWellKnownPollPeriod !== undefined) {
+        this._clientWellKnownIntervalID =
+            setInterval(() => {
+                this._fetchClientWellKnown();
+            }, 1000 * opts.clientWellKnownPollPeriod);
+        this._fetchClientWellKnown();
+    }
+};
+
+MatrixClient.prototype._fetchClientWellKnown = async function() {
+    // `getRawClientConfig` does not throw or reject on network errors, instead
+    // it absorbs errors and returns `{}`.
+    this._clientWellKnownPromise = AutoDiscovery.getRawClientConfig(
+        this.getDomain(),
+    );
+    this._clientWellKnown = await this._clientWellKnownPromise;
+    this.emit("WellKnown.client", this._clientWellKnown);
+};
+
+MatrixClient.prototype.getClientWellKnown = function() {
+    return this._clientWellKnown;
+};
+
+MatrixClient.prototype.waitForClientWellKnown = function() {
+    return this._clientWellKnownPromise;
 };

 /**
@@ -4799,6 +4971,9 @@ MatrixClient.prototype.stopClient = function() {
        this._peekSync.stopPeeking();
    }
    global.clearTimeout(this._checkTurnServersTimeoutID);
+    if (this._clientWellKnownIntervalID !== undefined) {
+        global.clearInterval(this._clientWellKnownIntervalID);
+    }
 };

 /**
@@ -5204,9 +5379,6 @@ function checkTurnServers(client) {
    if (!client._supportsVoip) {
        return;
    }
-    if (client.isGuest()) {
-        return; // guests can't access TURN servers
-    }

    client.turnServer().then(function(res) {
        if (res.uris) {
@@ -5247,17 +5419,20 @@ function _resolve(callback, resolve, res) {
    resolve(res);
 }

-function _PojoToMatrixEventMapper(client) {
+function _PojoToMatrixEventMapper(client, options) {
+    const preventReEmit = Boolean(options && options.preventReEmit);
    function mapper(plainOldJsObject) {
        const event = new MatrixEvent(plainOldJsObject);
        if (event.isEncrypted()) {
-            client.reEmitter.reEmit(event, [
-                "Event.decrypted",
-            ]);
+            if (!preventReEmit) {
+                client.reEmitter.reEmit(event, [
+                    "Event.decrypted",
+                ]);
+            }
            event.attemptDecryption(client._crypto);
        }
        const room = client.getRoom(event.getRoomId());
-        if (room) {
+        if (room && !preventReEmit) {
            room.reEmitter.reEmit(event, ["Event.replaced"]);
        }
        return event;
@@ -5266,10 +5441,12 @@ function _PojoToMatrixEventMapper(client) {
 }

 /**
+ * @param {object} [options]
+ * @param {bool} options.preventReEmit don't reemit events emitted on an event mapped by this mapper on the client
 * @return {Function}
 */
-MatrixClient.prototype.getEventMapper = function() {
-    return _PojoToMatrixEventMapper(this);
+MatrixClient.prototype.getEventMapper = function(options = undefined) {
+    return _PojoToMatrixEventMapper(this, options);
 };

 /**
@@ -5549,12 +5726,29 @@ MatrixClient.prototype.generateClientSecret = function() {
 * Fires whenever new user-scoped account_data is added.
 * @event module:client~MatrixClient#"accountData"
 * @param {MatrixEvent} event The event describing the account_data just added
+ * @param {MatrixEvent} event The previous account data, if known.
 * @example
- * matrixClient.on("accountData", function(event){
+ * matrixClient.on("accountData", function(event, oldEvent){
 *   myAccountData[event.type] = event.content;
 * });
 */

+/**
+ * Fires whenever the stored devices for a user have changed
+ * @event module:client~MatrixClient#"crypto.devicesUpdated"
+ * @param {String[]} users A list of user IDs that were updated
+ * @param {bool} initialFetch If true, the store was empty (apart
+ *     from our own device) and has been seeded.
+ */
+
+/**
+ * Fires whenever the stored devices for a user will be updated
+ * @event module:client~MatrixClient#"crypto.willUpdateDevices"
+ * @param {String[]} users A list of user IDs that will be updated
+ * @param {bool} initialFetch If true, the store is empty (apart
+ *     from our own device) and is being seeded.
+ */
+
 /**
 * Fires whenever the status of e2e key backup changes, as returned by getKeyBackupEnabled()
 * @event module:client~MatrixClient#"crypto.keyBackupStatus"
@@ -5612,6 +5806,13 @@ MatrixClient.prototype.generateClientSecret = function() {
 * @param {string} data.request_id The ID of the original request.
 */

+/**
+ * Fires when the client .well-known info is fetched.
+ *
+ * @event module:client~MatrixClient#"WellKnown.client"
+ * @param {object} data The JSON object returned by the server
+ */
+
 // EventEmitter JSDocs

 /**
@@ -75,35 +75,3 @@ export function getHttpUriForMxc(baseUrl, mxc, width, height,
        (utils.keys(params).length === 0 ? "" :
        ("?" + utils.encodeParams(params))) + fragment;
 }
-
-/**
- * Get an identicon URL from an arbitrary string.
- * @param {string} baseUrl The base homeserver url which has a content repo.
- * @param {string} identiconString The string to create an identicon for.
- * @param {Number} width The desired width of the image in pixels. Default: 96.
- * @param {Number} height The desired height of the image in pixels. Default: 96.
- * @return {string} The complete URL to the identicon.
- * @deprecated This is no longer in the specification.
- */
-export function getIdenticonUri(baseUrl, identiconString, width, height) {
-    if (!identiconString) {
-        return null;
-    }
-    if (!width) {
-        width = 96;
-    }
-    if (!height) {
-        height = 96;
-    }
-    const params = {
-        width: width,
-        height: height,
-    };
-
-    const path = utils.encodeUri("/_matrix/media/unstable/identicon/$ident", {
-        $ident: identiconString,
-    });
-    return baseUrl + path +
-        (utils.keys(params).length === 0 ? "" :
-            ("?" + utils.encodeParams(params)));
-}
@@ -24,6 +24,9 @@ import {decodeBase64, encodeBase64, pkSign, pkVerify} from './olmlib';
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
 import {IndexedDBCryptoStore} from '../crypto/store/indexeddb-crypto-store';
+import {decryptAES, encryptAES} from './aes';
+
+const KEY_REQUEST_TIMEOUT_MS = 1000 * 60;

 function publicKeyFromKeyInfo(keyInfo) {
    // `keys` is an object with { [`ed25519:${pubKey}`]: pubKey }
@@ -55,17 +58,42 @@ export class CrossSigningInfo extends EventEmitter {
        this._cacheCallbacks = cacheCallbacks || {};
        this.keys = {};
        this.firstUse = true;
+        // This tracks whether we've ever verified this user with any identity.
+        // When you verify a user, any devices online at the time that receive
+        // the verifying signature via the homeserver will latch this to true
+        // and can use it in the future to detect cases where the user has
+        // become unverifed later for any reason.
+        this.crossSigningVerifiedBefore = false;
+    }
+
+    static fromStorage(obj, userId) {
+        const res = new CrossSigningInfo(userId);
+        for (const prop in obj) {
+            if (obj.hasOwnProperty(prop)) {
+                res[prop] = obj[prop];
+            }
+        }
+        return res;
+    }
+
+    toStorage() {
+        return {
+            keys: this.keys,
+            firstUse: this.firstUse,
+            crossSigningVerifiedBefore: this.crossSigningVerifiedBefore,
+        };
    }

    /**
     * Calls the app callback to ask for a private key
+     *
     * @param {string} type The key type ("master", "self_signing", or "user_signing")
     * @param {string} expectedPubkey The matching public key or undefined to use
     *     the stored public key for the given key type.
     * @returns {Array} An array with [ public key, Olm.PkSigning ]
     */
    async getCrossSigningKey(type, expectedPubkey) {
-        const shouldCache = ["self_signing", "user_signing"].indexOf(type) >= 0;
+        const shouldCache = ["master", "self_signing", "user_signing"].indexOf(type) >= 0;

        if (!this._callbacks.getCrossSigningKey) {
            throw new Error("No getCrossSigningKey callback supplied");
@@ -118,23 +146,6 @@ export class CrossSigningInfo extends EventEmitter {
        );
    }

-    static fromStorage(obj, userId) {
-        const res = new CrossSigningInfo(userId);
-        for (const prop in obj) {
-            if (obj.hasOwnProperty(prop)) {
-                res[prop] = obj[prop];
-            }
-        }
-        return res;
-    }
-
-    toStorage() {
-        return {
-            keys: this.keys,
-            firstUse: this.firstUse,
-        };
-    }
-
    /**
     * Check whether the private keys exist in secret storage.
     * XXX: This could be static, be we often seem to have an instance when we
@@ -170,12 +181,12 @@ export class CrossSigningInfo extends EventEmitter {
     * typically called in conjunction with the creation of new cross-signing
     * keys.
     *
-     * @param {object} keys The keys to store
+     * @param {Map} keys The keys to store
     * @param {SecretStorage} secretStorage The secret store using account data
     */
    static async storeInSecretStorage(keys, secretStorage) {
-        for (const type of Object.keys(keys)) {
-            const encodedKey = encodeBase64(keys[type]);
+        for (const [type, privateKey] of keys) {
+            const encodedKey = encodeBase64(privateKey);
            await secretStorage.store(`m.cross_signing.${type}`, encodedKey);
        }
    }
@@ -191,9 +202,44 @@ export class CrossSigningInfo extends EventEmitter {
     */
    static async getFromSecretStorage(type, secretStorage) {
        const encodedKey = await secretStorage.get(`m.cross_signing.${type}`);
+        if (!encodedKey) {
+            return null;
+        }
        return decodeBase64(encodedKey);
    }

+    /**
+     * Check whether the private keys exist in the local key cache.
+     *
+     * @returns {boolean} True if all keys are stored in the local cache.
+     */
+    async isStoredInKeyCache() {
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return false;
+        for (const type of ["master", "self_signing", "user_signing"]) {
+            if (!await cacheCallbacks.getCrossSigningKeyCache(type)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    /**
+     * Get cross-signing private keys from the local cache.
+     *
+     * @returns {Map} A map from key type (string) to private key (Uint8Array)
+     */
+    async getCrossSigningKeysFromCache() {
+        const keys = new Map();
+        const cacheCallbacks = this._cacheCallbacks;
+        if (!cacheCallbacks) return keys;
+        for (const type of ["master", "self_signing", "user_signing"]) {
+            const privKey = await cacheCallbacks.getCrossSigningKeyCache(type);
+            keys.set(type, privKey);
+        }
+        return keys;
+    }
+
    /**
     * Get the ID used to identify the user. This can also be used to test for
     * the existence of a given key type.
@@ -303,6 +349,13 @@ export class CrossSigningInfo extends EventEmitter {
        }
    }

+    /**
+     * unsets the keys, used when another session has reset the keys, to disable cross-signing
+     */
+    clearKeys() {
+        this.keys = {};
+    }
+
    setKeys(keys) {
        const signingKeys = {};
        if (keys.master) {
@@ -375,6 +428,14 @@ export class CrossSigningInfo extends EventEmitter {
        }
    }

+    updateCrossSigningVerifiedBefore(isCrossSigningVerified) {
+        // It is critical that this value latches forward from false to true but
+        // never back to false to avoid a downgrade attack.
+        if (!this.crossSigningVerifiedBefore && isCrossSigningVerified) {
+            this.crossSigningVerifiedBefore = true;
+        }
+    }
+
    async signObject(data, type) {
        if (!this.keys[type]) {
            throw new Error(
@@ -433,13 +494,13 @@ export class CrossSigningInfo extends EventEmitter {
            && this.getId("self_signing")
            && this.getId("self_signing") === userCrossSigning.getId("self_signing")
        ) {
-            return new UserTrustLevel(true, this.firstUse);
+            return new UserTrustLevel(true, true, this.firstUse);
        }

        if (!this.keys.user_signing) {
            // If there's no user signing key, they can't possibly be verified.
            // They may be TOFU trusted though.
-            return new UserTrustLevel(false, userCrossSigning.firstUse);
+            return new UserTrustLevel(false, false, userCrossSigning.firstUse);
        }

        let userTrusted;
@@ -451,7 +512,11 @@ export class CrossSigningInfo extends EventEmitter {
        } catch (e) {
            userTrusted = false;
        }
-        return new UserTrustLevel(userTrusted, userCrossSigning.firstUse);
+        return new UserTrustLevel(
+            userTrusted,
+            userCrossSigning.crossSigningVerifiedBefore,
+            userCrossSigning.firstUse,
+        );
    }

    /**
@@ -523,8 +588,9 @@ export const CrossSigningLevel = {
 * Represents the ways in which we trust a user
 */
 export class UserTrustLevel {
-    constructor(crossSigningVerified, tofu) {
+    constructor(crossSigningVerified, crossSigningVerifiedBefore, tofu) {
        this._crossSigningVerified = crossSigningVerified;
+        this._crossSigningVerifiedBefore = crossSigningVerifiedBefore;
        this._tofu = tofu;
    }

@@ -542,6 +608,14 @@ export class UserTrustLevel {
        return this._crossSigningVerified;
    }

+    /**
+     * @returns {bool} true if we ever verified this user before (at least for
+     * the history of verifications observed by this device).
+     */
+    wasCrossSigningVerified() {
+        return this._crossSigningVerifiedBefore;
+    }
+
    /**
     * @returns {bool} true if this user's key is trusted on first use
     */
@@ -602,10 +676,10 @@ export class DeviceTrustLevel {
    }
 }

-export function createCryptoStoreCacheCallbacks(store) {
+export function createCryptoStoreCacheCallbacks(store, olmdevice) {
    return {
-        getCrossSigningKeyCache: function(type, _expectedPublicKey) {
-            return new Promise((resolve) => {
+        getCrossSigningKeyCache: async function(type, _expectedPublicKey) {
+            const key = await new Promise((resolve) => {
                return store.doTxn(
                    'readonly',
                    [IndexedDBCryptoStore.STORE_ACCOUNT],
@@ -614,8 +688,23 @@ export function createCryptoStoreCacheCallbacks(store) {
                    },
                );
            });
+
+            if (key && key.ciphertext) {
+                const pickleKey = Buffer.from(olmdevice._pickleKey);
+                const decrypted = await decryptAES(key, pickleKey, type);
+                return decodeBase64(decrypted);
+            } else {
+                return key;
+            }
        },
-        storeCrossSigningKeyCache: function(type, key) {
+        storeCrossSigningKeyCache: async function(type, key) {
+            if (!(key instanceof Uint8Array)) {
+                throw new Error(
+                    `storeCrossSigningKeyCache expects Uint8Array, got ${key}`,
+                );
+            }
+            const pickleKey = Buffer.from(olmdevice._pickleKey);
+            key = await encryptAES(encodeBase64(key), pickleKey, type);
            return store.doTxn(
                'readwrite',
                [IndexedDBCryptoStore.STORE_ACCOUNT],
@@ -626,3 +715,96 @@ export function createCryptoStoreCacheCallbacks(store) {
        },
    };
 }
+
+/**
+ * Request cross-signing keys from another device during verification.
+ *
+ * @param {module:base-apis~MatrixBaseApis} baseApis base Matrix API interface
+ * @param {string} userId The user ID being verified
+ * @param {string} deviceId The device ID being verified
+ */
+export async function requestKeysDuringVerification(baseApis, userId, deviceId) {
+    // If this is a self-verification, ask the other party for keys
+    if (baseApis.getUserId() !== userId) {
+        return;
+    }
+    console.log("Cross-signing: Self-verification done; requesting keys");
+    // This happens asynchronously, and we're not concerned about waiting for
+    // it.  We return here in order to test.
+    return new Promise((resolve, reject) => {
+        const client = baseApis;
+        const original = client._crypto._crossSigningInfo;
+
+        // We already have all of the infrastructure we need to validate and
+        // cache cross-signing keys, so instead of replicating that, here we set
+        // up callbacks that request them from the other device and call
+        // CrossSigningInfo.getCrossSigningKey() to validate/cache
+        const crossSigning = new CrossSigningInfo(
+            original.userId,
+            { getCrossSigningKey: async (type) => {
+                console.debug("Cross-signing: requesting secret",
+                                type, deviceId);
+                const { promise } = client.requestSecret(
+                    `m.cross_signing.${type}`, [deviceId],
+                );
+                const result = await promise;
+                const decoded = decodeBase64(result);
+                return Uint8Array.from(decoded);
+            } },
+            original._cacheCallbacks,
+        );
+        crossSigning.keys = original.keys;
+
+        // XXX: get all keys out if we get one key out
+        // https://github.com/vector-im/element-web/issues/12604
+        // then change here to reject on the timeout
+        // Requests can be ignored, so don't wait around forever
+        const timeout = new Promise((resolve, reject) => {
+            setTimeout(
+                resolve,
+                KEY_REQUEST_TIMEOUT_MS,
+                new Error("Timeout"),
+            );
+        });
+
+        // also request and cache the key backup key
+        const backupKeyPromise = new Promise(async resolve => {
+            const cachedKey = await client._crypto.getSessionBackupPrivateKey();
+            if (!cachedKey) {
+                logger.info("No cached backup key found. Requesting...");
+                const secretReq = client.requestSecret(
+                    'm.megolm_backup.v1', [deviceId],
+                );
+                const base64Key = await secretReq.promise;
+                logger.info("Got key backup key, decoding...");
+                const decodedKey = decodeBase64(base64Key);
+                logger.info("Decoded backup key, storing...");
+                client._crypto.storeSessionBackupPrivateKey(
+                    Uint8Array.from(decodedKey),
+                );
+                logger.info("Backup key stored. Starting backup restore...");
+                const backupInfo = await client.getKeyBackupVersion();
+                // no need to await for this - just let it go in the bg
+                client.restoreKeyBackupWithCache(
+                    undefined, undefined, backupInfo,
+                ).then(() => {
+                    logger.info("Backup restored.");
+                });
+            }
+            resolve();
+        });
+
+        // We call getCrossSigningKey() for its side-effects
+        return Promise.race([
+            Promise.all([
+                crossSigning.getCrossSigningKey("master"),
+                crossSigning.getCrossSigningKey("self_signing"),
+                crossSigning.getCrossSigningKey("user_signing"),
+                backupKeyPromise,
+            ]),
+            timeout,
+        ]).then(resolve, reject);
+    }).catch((e) => {
+        console.warn("Cross-signing: failure while requesting keys:", e);
+    });
+}
@@ -109,6 +109,9 @@ export class DeviceList extends EventEmitter {
        this._savePromiseTime = null;
        // The timer used to delay the save
        this._saveTimer = null;
+        // True if we have fetched data from the server or loaded a non-empty
+        // set of device data from the store
+        this._hasFetched = null;
    }

    /**
@@ -118,6 +121,7 @@ export class DeviceList extends EventEmitter {
        await this._cryptoStore.doTxn(
            'readonly', [IndexedDBCryptoStore.STORE_DEVICE_DATA], (txn) => {
                this._cryptoStore.getEndToEndDeviceData(txn, (deviceData) => {
+                    this._hasFetched = Boolean(deviceData && deviceData.devices);
                    this._devices = deviceData ? deviceData.devices : {},
                    this._crossSigningInfo = deviceData ?
                        deviceData.crossSigningInfo || {} : {};
@@ -652,6 +656,7 @@ export class DeviceList extends EventEmitter {
        });

        const finished = (success) => {
+            this.emit("crypto.willUpdateDevices", users, !this._hasFetched);
            users.forEach((u) => {
                this._dirty = true;

@@ -677,7 +682,8 @@ export class DeviceList extends EventEmitter {
                }
            });
            this.saveIfDirty();
-            this.emit("crypto.devicesUpdated", users);
+            this.emit("crypto.devicesUpdated", users, !this._hasFetched);
+            this._hasFetched = true;
        };

        return prom;
@@ -784,7 +790,7 @@ class DeviceListUpdateSerialiser {

            // yield to other things that want to execute in between users, to
            // avoid wedging the CPU
-            // (https://github.com/vector-im/riot-web/issues/3158)
+            // (https://github.com/vector-im/element-web/issues/3158)
            //
            // of course we ought to do this in a web worker or similar, but
            // this serves as an easy solution for now.
@@ -824,7 +830,7 @@ class DeviceListUpdateSerialiser {
    }

    async _processQueryResponseForUser(
-        userId, dkResponse, crossSigningResponse, sskResponse,
+        userId, dkResponse, crossSigningResponse,
    ) {
        logger.log('got device keys for ' + userId + ':', dkResponse);
        logger.log('got cross-signing keys for ' + userId + ':', crossSigningResponse);
@@ -0,0 +1,357 @@
+import { logger } from "../logger";
+import {MatrixEvent} from "../models/event";
+import {EventEmitter} from "events";
+import {createCryptoStoreCacheCallbacks} from "./CrossSigning";
+import {IndexedDBCryptoStore} from './store/indexeddb-crypto-store';
+import {
+    PREFIX_UNSTABLE,
+} from "../http-api";
+
+/**
+ * Builds an EncryptionSetupOperation by calling any of the add.. methods.
+ * Once done, `buildOperation()` can be called which allows to apply to operation.
+ *
+ * This is used as a helper by Crypto to keep track of all the network requests
+ * and other side-effects of bootstrapping, so it can be applied in one go (and retried in the future)
+ * Also keeps track of all the private keys created during bootstrapping, so we don't need to prompt for them
+ * more than once.
+ */
+export class EncryptionSetupBuilder {
+    /**
+     * @param {Object.<String, MatrixEvent>} accountData pre-existing account data, will only be read, not written.
+     * @param {CryptoCallbacks} delegateCryptoCallbacks crypto callbacks to delegate to if the key isn't in cache yet
+     */
+    constructor(accountData, delegateCryptoCallbacks) {
+        this.accountDataClientAdapter = new AccountDataClientAdapter(accountData);
+        this.crossSigningCallbacks = new CrossSigningCallbacks();
+        this.ssssCryptoCallbacks = new SSSSCryptoCallbacks(delegateCryptoCallbacks);
+
+        this._crossSigningKeys = null;
+        this._keySignatures = null;
+        this._keyBackupInfo = null;
+    }
+
+    /**
+     * Adds new cross-signing public keys
+     *
+     * @param {function} authUpload Function called to await an interactive auth
+     * flow when uploading device signing keys.
+     * Args:
+     *     {function} A function that makes the request requiring auth. Receives
+     *     the auth data as an object. Can be called multiple times, first with
+     *     an empty authDict, to obtain the flows.
+     * @param {Object} keys the new keys
+     */
+    addCrossSigningKeys(authUpload, keys) {
+        this._crossSigningKeys = {authUpload, keys};
+    }
+
+    /**
+     * Adds the key backup info to be updated on the server
+     *
+     * Used either to create a new key backup, or add signatures
+     * from the new MSK.
+     *
+     * @param {Object} keyBackupInfo as received from/sent to the server
+     */
+    addSessionBackup(keyBackupInfo) {
+        this._keyBackupInfo = keyBackupInfo;
+    }
+
+    /**
+     * Adds the session backup private key to be updated in the local cache
+     *
+     * Used after fixing the format of the key
+     *
+     * @param {Uint8Array} privateKey
+     */
+    addSessionBackupPrivateKeyToCache(privateKey) {
+        this._sessionBackupPrivateKey = privateKey;
+    }
+
+    /**
+     * Add signatures from a given user and device/x-sign key
+     * Used to sign the new cross-signing key with the device key
+     *
+     * @param {String} userId
+     * @param {String} deviceId
+     * @param {String} signature
+     */
+    addKeySignature(userId, deviceId, signature) {
+        if (!this._keySignatures) {
+            this._keySignatures = {};
+        }
+        const userSignatures = this._keySignatures[userId] || {};
+        this._keySignatures[userId] = userSignatures;
+        userSignatures[deviceId] = signature;
+    }
+
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        return this.accountDataClientAdapter.setAccountData(type, content);
+    }
+
+    /**
+     * builds the operation containing all the parts that have been added to the builder
+     * @return {EncryptionSetupOperation}
+     */
+    buildOperation() {
+        const accountData = this.accountDataClientAdapter._values;
+        return new EncryptionSetupOperation(
+            accountData,
+            this._crossSigningKeys,
+            this._keyBackupInfo,
+            this._keySignatures,
+        );
+    }
+
+    /**
+     * Stores the created keys locally.
+     *
+     * This does not yet store the operation in a way that it can be restored,
+     * but that is the idea in the future.
+     *
+     * @param  {Crypto} crypto
+     * @return {Promise}
+     */
+    async persist(crypto) {
+        // store private keys in cache
+        if (this._crossSigningKeys) {
+            const cacheCallbacks = createCryptoStoreCacheCallbacks(
+                crypto._cryptoStore, crypto._olmDevice);
+            for (const type of ["master", "self_signing", "user_signing"]) {
+                logger.log(`Cache ${type} cross-signing private key locally`);
+                const privateKey = this.crossSigningCallbacks.privateKeys.get(type);
+                await cacheCallbacks.storeCrossSigningKeyCache(type, privateKey);
+            }
+            // store own cross-sign pubkeys as trusted
+            await crypto._cryptoStore.doTxn(
+                'readwrite', [IndexedDBCryptoStore.STORE_ACCOUNT],
+                (txn) => {
+                    crypto._cryptoStore.storeCrossSigningKeys(
+                        txn, this._crossSigningKeys.keys);
+                },
+            );
+        }
+        // store session backup key in cache
+        if (this._sessionBackupPrivateKey) {
+            await crypto.storeSessionBackupPrivateKey(this._sessionBackupPrivateKey);
+        }
+    }
+}
+
+/**
+ * Can be created from EncryptionSetupBuilder, or
+ * (in a follow-up PR, not implemented yet) restored from storage, to retry.
+ *
+ * It does not have knowledge of any private keys, unlike the builder.
+ */
+export class EncryptionSetupOperation {
+    /**
+     * @param  {Map<String, Object>} accountData
+     * @param  {Object} crossSigningKeys
+     * @param  {Object} keyBackupInfo
+     * @param  {Object} keySignatures
+     */
+    constructor(accountData, crossSigningKeys, keyBackupInfo, keySignatures) {
+        this._accountData = accountData;
+        this._crossSigningKeys = crossSigningKeys;
+        this._keyBackupInfo = keyBackupInfo;
+        this._keySignatures = keySignatures;
+    }
+
+    /**
+     * Runs the (remaining part of, in the future) operation by sending requests to the server.
+     * @param  {Crypto} crypto
+     */
+    async apply(crypto) {
+        const baseApis = crypto._baseApis;
+        // upload cross-signing keys
+        if (this._crossSigningKeys) {
+            const keys = {};
+            for (const [name, key] of Object.entries(this._crossSigningKeys.keys)) {
+                keys[name + "_key"] = key;
+            }
+
+            // We must only call `uploadDeviceSigningKeys` from inside this auth
+            // helper to ensure we properly handle auth errors.
+            await this._crossSigningKeys.authUpload(authDict => {
+                return baseApis.uploadDeviceSigningKeys(authDict, keys);
+            });
+
+            // pass the new keys to the main instance of our own CrossSigningInfo.
+            crypto._crossSigningInfo.setKeys(this._crossSigningKeys.keys);
+        }
+        // set account data
+        if (this._accountData) {
+            for (const [type, content] of this._accountData) {
+                await baseApis.setAccountData(type, content);
+            }
+        }
+        // upload first cross-signing signatures with the new key
+        // (e.g. signing our own device)
+        if (this._keySignatures) {
+            await baseApis.uploadKeySignatures(this._keySignatures);
+        }
+        // need to create/update key backup info
+        if (this._keyBackupInfo) {
+            if (this._keyBackupInfo.version) {
+                // session backup signature
+                // The backup is trusted because the user provided the private key.
+                // Sign the backup with the cross signing key so the key backup can
+                // be trusted via cross-signing.
+                await baseApis._http.authedRequest(
+                    undefined, "PUT", "/room_keys/version/" + this._keyBackupInfo.version,
+                    undefined, {
+                        algorithm: this._keyBackupInfo.algorithm,
+                        auth_data: this._keyBackupInfo.auth_data,
+                    },
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            } else {
+                // add new key backup
+                await baseApis._http.authedRequest(
+                    undefined, "POST", "/room_keys/version",
+                    undefined, this._keyBackupInfo,
+                    {prefix: PREFIX_UNSTABLE},
+                );
+            }
+        }
+    }
+}
+
+
+/**
+ * Catches account data set by SecretStorage during bootstrapping by
+ * implementing the methods related to account data in MatrixClient
+ */
+class AccountDataClientAdapter extends EventEmitter {
+    /**
+     * @param  {Object.<String, MatrixEvent>} accountData existing account data
+     */
+    constructor(accountData) {
+        super();
+        this._existingValues = accountData;
+        this._values = new Map();
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Promise<Object>} the content of the account data
+     */
+    getAccountDataFromServer(type) {
+        return Promise.resolve(this.getAccountData(type));
+    }
+
+    /**
+     * @param  {String} type
+     * @return {Object} the content of the account data
+     */
+    getAccountData(type) {
+        const modifiedValue = this._values.get(type);
+        if (modifiedValue) {
+            return modifiedValue;
+        }
+        const existingValue = this._existingValues[type];
+        if (existingValue) {
+            return existingValue.getContent();
+        }
+        return null;
+    }
+
+    /**
+     * @param {String} type
+     * @param {Object} content
+     * @return {Promise}
+     */
+    setAccountData(type, content) {
+        const lastEvent = this._values.get(type);
+        this._values.set(type, content);
+        // ensure accountData is emitted on the next tick,
+        // as SecretStorage listens for it while calling this method
+        // and it seems to rely on this.
+        return Promise.resolve().then(() => {
+            const event = new MatrixEvent({type, content});
+            this.emit("accountData", event, lastEvent);
+        });
+    }
+}
+
+/**
+ * Catches the private cross-signing keys set during bootstrapping
+ * by both cache callbacks (see createCryptoStoreCacheCallbacks) as non-cache callbacks.
+ * See CrossSigningInfo constructor
+ */
+class CrossSigningCallbacks {
+    constructor() {
+        this.privateKeys = new Map();
+    }
+
+    // cache callbacks
+    getCrossSigningKeyCache(type, expectedPublicKey) {
+        return this.getCrossSigningKey(type, expectedPublicKey);
+    }
+
+    storeCrossSigningKeyCache(type, key) {
+        this.privateKeys.set(type, key);
+        return Promise.resolve();
+    }
+
+    // non-cache callbacks
+    getCrossSigningKey(type, _expectedPubkey) {
+        return Promise.resolve(this.privateKeys.get(type));
+    }
+
+    saveCrossSigningKeys(privateKeys) {
+        for (const [type, privateKey] of Object.entries(privateKeys)) {
+            this.privateKeys.set(type, privateKey);
+        }
+    }
+}
+
+/**
+ * Catches the 4S private key set during bootstrapping by implementing
+ * the SecretStorage crypto callbacks
+ */
+class SSSSCryptoCallbacks {
+    constructor(delegateCryptoCallbacks) {
+        this._privateKeys = new Map();
+        this._delegateCryptoCallbacks = delegateCryptoCallbacks;
+    }
+
+    async getSecretStorageKey({ keys }, name) {
+        for (const keyId of Object.keys(keys)) {
+            const privateKey = this._privateKeys.get(keyId);
+            if (privateKey) {
+                return [keyId, privateKey];
+            }
+        }
+        // if we don't have the key cached yet, ask
+        // for it to the general crypto callbacks and cache it
+        if (this._delegateCryptoCallbacks) {
+            const result = await this._delegateCryptoCallbacks.
+                getSecretStorageKey({keys}, name);
+            if (result) {
+                const [keyId, privateKey] = result;
+                this._privateKeys.set(keyId, privateKey);
+            }
+            return result;
+        }
+    }
+
+    addPrivateKey(keyId, privKey) {
+        this._privateKeys.set(keyId, privKey);
+        // Also pass along to application to cache if it wishes
+        if (
+            this._delegateCryptoCallbacks &&
+            this._delegateCryptoCallbacks.cacheSecretStorageKey
+        ) {
+            this._delegateCryptoCallbacks.cacheSecretStorageKey(keyId, privKey);
+        }
+    }
+}
@@ -36,9 +36,15 @@ function checkPayloadLength(payloadString) {
        // Note that even if we manage to do the encryption, the message send may fail,
        // because by the time we've wrapped the ciphertext in the event object, it may
        // exceed 65K. But at least we won't just fail with "abort()" in that case.
-        throw new Error("Message too long (" + payloadString.length + " bytes). " +
+        const err = new Error("Message too long (" + payloadString.length + " bytes). " +
                        "The maximum for an encrypted message is " +
                        MAX_PLAINTEXT_LENGTH + " bytes.");
+        // TODO: [TypeScript] We should have our own error types
+        err.data = {
+            errcode: "M_TOO_LARGE",
+            error: "Payload too large for encrypted message",
+        };
+        throw err;
    }
 }

@@ -105,6 +111,9 @@ export function OlmDevice(cryptoStore) {
    // Keep track of sessions that we're starting, so that we don't start
    // multiple sessions for the same device at the same time.
    this._sessionsInProgress = {};
+
+    // Used by olm to serialise prekey message decryptions
+    this._olmPrekeyPromise = Promise.resolve();
 }

 /**
@@ -983,11 +992,12 @@ OlmDevice.prototype._getInboundGroupSession = function(
 * @param {Object<string, string>} keysClaimed Other keys the sender claims.
 * @param {boolean} exportFormat true if the megolm keys are in export format
 *    (ie, they lack an ed25519 signature)
+ * @param {Object} [extraSessionData={}] any other data to be include with the session
 */
 OlmDevice.prototype.addInboundGroupSession = async function(
    roomId, senderKey, forwardingCurve25519KeyChain,
    sessionId, sessionKey, keysClaimed,
-    exportFormat,
+    exportFormat, extraSessionData = {},
 ) {
    await this._cryptoStore.doTxn(
        'readwrite', [
@@ -1029,12 +1039,17 @@ OlmDevice.prototype.addInboundGroupSession = async function(
                            }
                        }

-                        const sessionData = {
+                        logger.info(
+                            "Storing megolm session " + senderKey + "/" + sessionId +
+                            " with first index " + session.first_known_index(),
+                        );
+
+                        const sessionData = Object.assign({}, extraSessionData, {
                            room_id: roomId,
                            session: session.pickle(this._pickleKey),
                            keysClaimed: keysClaimed,
                            forwardingCurve25519KeyChain: forwardingCurve25519KeyChain,
-                        };
+                        });

                        this._cryptoStore.storeEndToEndInboundGroupSession(
                            senderKey, sessionId, sessionData, txn,
@@ -1210,6 +1225,7 @@ OlmDevice.prototype.decryptGroupMessage = async function(
                        forwardingCurve25519KeyChain: (
                            sessionData.forwardingCurve25519KeyChain || []
                        ),
+                        untrusted: sessionData.untrusted,
                    };
                },
            );
@@ -58,7 +58,7 @@ const SEND_KEY_REQUESTS_DELAY_MS = 500;
 *
 * @enum {number}
 */
-const ROOM_KEY_REQUEST_STATES = {
+export const ROOM_KEY_REQUEST_STATES = {
    /** request not yet sent */
    UNSENT: 0,

@@ -97,10 +97,6 @@ export class OutgoingRoomKeyRequestManager {
     */
    start() {
        this._clientRunning = true;
-
-        // set the timer going, to handle any requests which didn't get sent
-        // on the previous run of the client.
-        this._startTimer();
    }

    /**
@@ -113,7 +109,14 @@ export class OutgoingRoomKeyRequestManager {
    }

    /**
-     * Send off a room key request, if we haven't already done so.
+     * Send any requests that have been queued
+     */
+    sendQueuedRequests() {
+        this._startTimer();
+    }
+
+    /**
+     * Queue up a room key request, if we haven't already queued or sent one.
     *
     * The `requestBody` is compared (with a deep-equality check) against
     * previous queued or sent requests and if it matches, no change is made.
@@ -129,7 +132,7 @@ export class OutgoingRoomKeyRequestManager {
     *    pending list (or we have established that a similar request already
     *    exists)
     */
-    async sendRoomKeyRequest(requestBody, recipients, resend=false) {
+    async queueRoomKeyRequest(requestBody, recipients, resend=false) {
        const req = await this._cryptoStore.getOutgoingRoomKeyRequest(
            requestBody,
        );
@@ -184,7 +187,7 @@ export class OutgoingRoomKeyRequestManager {
                        // in state ROOM_KEY_REQUEST_STATES.SENT, so we must have
                        // raced with another tab to mark the request cancelled.
                        // Try again, to make sure the request is resent.
-                        return await this.sendRoomKeyRequest(
+                        return await this.queueRoomKeyRequest(
                            requestBody, recipients, resend,
                        );
                    }
@@ -220,9 +223,6 @@ export class OutgoingRoomKeyRequestManager {
                throw new Error('unhandled state: ' + req.state);
            }
        }
-        // some of the branches require the timer to be started.  Just start it
-        // all the time, because it doesn't hurt to start it.
-        this._startTimer();
    }

    /**
@@ -327,6 +327,21 @@ export class OutgoingRoomKeyRequestManager {
        );
    }

+    /**
+     * Find anything in `sent` state, and kick it around the loop again.
+     * This is intended for situations where something substantial has changed, and we
+     * don't really expect the other end to even care about the cancellation.
+     * For example, after initialization or self-verification.
+     * @return {Promise} An array of `queueRoomKeyRequest` outputs.
+     */
+    async cancelAndResendAllOutgoingRequests() {
+        const outgoings = await this._cryptoStore.getAllOutgoingRoomKeyRequestsByState(
+            ROOM_KEY_REQUEST_STATES.SENT,
+        );
+        return Promise.all(outgoings.map(({ requestBody, recipients }) =>
+            this.queueRoomKeyRequest(requestBody, recipients, true)));
+    }
+
    // start the background timer to send queued requests, if the timer isn't
    // already running
    _startTimer() {
@@ -366,15 +381,12 @@ export class OutgoingRoomKeyRequestManager {
            return Promise.resolve();
        }

-        logger.log("Looking for queued outgoing room key requests");
-
        return this._cryptoStore.getOutgoingRoomKeyRequestByState([
            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING,
            ROOM_KEY_REQUEST_STATES.CANCELLATION_PENDING_AND_WILL_RESEND,
            ROOM_KEY_REQUEST_STATES.UNSENT,
        ]).then((req) => {
            if (!req) {
-                logger.log("No more outgoing room key requests");
                this._sendOutgoingRoomKeyRequestsTimer = null;
                return;
            }
@@ -398,7 +410,6 @@ export class OutgoingRoomKeyRequestManager {
            }).catch((e) => {
                logger.error("Error sending room key request; will retry later.", e);
                this._sendOutgoingRoomKeyRequestsTimer = null;
-                this._startTimer();
            });
        });
    }
@@ -17,26 +17,24 @@ limitations under the License.
 import {EventEmitter} from 'events';
 import {logger} from '../logger';
 import * as olmlib from './olmlib';
-import {pkVerify} from './olmlib';
 import {randomString} from '../randomstring';
 import {encryptAES, decryptAES} from './aes';
+import {encodeBase64} from "./olmlib";

 export const SECRET_STORAGE_ALGORITHM_V1_AES
    = "m.secret_storage.v1.aes-hmac-sha2";
-// don't use curve25519 for writing data.
-export const SECRET_STORAGE_ALGORITHM_V1_CURVE25519
-    = "m.secret_storage.v1.curve25519-aes-sha2";
+
+const ZERO_STR = "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0";

 /**
 * Implements Secure Secret Storage and Sharing (MSC1946)
 * @module crypto/SecretStorage
 */
 export class SecretStorage extends EventEmitter {
-    constructor(baseApis, cryptoCallbacks, crossSigningInfo) {
+    constructor(baseApis, cryptoCallbacks) {
        super();
        this._baseApis = baseApis;
        this._cryptoCallbacks = cryptoCallbacks;
-        this._crossSigningInfo = crossSigningInfo;
        this._requests = {};
        this._incomingRequests = {};
    }
@@ -50,7 +48,7 @@ export class SecretStorage extends EventEmitter {
    }

    setDefaultKeyId(keyId) {
-        return new Promise((resolve) => {
+        return new Promise(async (resolve, reject) => {
            const listener = (ev) => {
                if (
                    ev.getType() === 'm.secret_storage.default_key' &&
@@ -62,10 +60,15 @@ export class SecretStorage extends EventEmitter {
            };
            this._baseApis.on('accountData', listener);

-            this._baseApis.setAccountData(
-                'm.secret_storage.default_key',
-                { key: keyId },
-            );
+            try {
+                await this._baseApis.setAccountData(
+                    'm.secret_storage.default_key',
+                    { key: keyId },
+                );
+            } catch (e) {
+                this._baseApis.removeListener('accountData', listener);
+                reject(e);
+            }
        });
    }

@@ -89,20 +92,16 @@ export class SecretStorage extends EventEmitter {
            keyData.name = opts.name;
        }

-        switch (algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1_AES:
-        {
-            const decryption = new global.Olm.PkDecryption();
-            try {
-                if (opts.passphrase) {
-                    keyData.passphrase = opts.passphrase;
-                }
-            } finally {
-                decryption.free();
+        if (algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (opts.passphrase) {
+                keyData.passphrase = opts.passphrase;
            }
-            break;
-        }
-        default:
+            if (opts.key) {
+                const {iv, mac} = await SecretStorage._calculateKeyCheck(opts.key);
+                keyData.iv = iv;
+                keyData.mac = mac;
+            }
+        } else {
            throw new Error(`Unknown key algorithm ${opts.algorithm}`);
        }

@@ -116,8 +115,6 @@ export class SecretStorage extends EventEmitter {
            );
        }

-        await this._crossSigningInfo.signObject(keyData, 'master');
-
        await this._baseApis.setAccountData(
            `m.secret_storage.key.${keyId}`, keyData,
        );
@@ -125,33 +122,6 @@ export class SecretStorage extends EventEmitter {
        return keyId;
    }

-    /**
-     * Signs a given secret storage key with the cross-signing master key.
-     *
-     * @param {string} [keyId = default key's ID] The ID of the key to sign.
-     *     Defaults to the default key ID if not provided.
-     */
-    async signKey(keyId) {
-        if (!keyId) {
-            keyId = await this.getDefaultKeyId();
-        }
-        if (!keyId) {
-            throw new Error("signKey requires a key ID");
-        }
-
-        const keyInfo = await this._baseApis.getAccountDataFromServer(
-            `m.secret_storage.key.${keyId}`,
-        );
-        if (!keyInfo) {
-            throw new Error(`Key ${keyId} does not exist in account data`);
-        }
-
-        await this._crossSigningInfo.signObject(keyInfo, 'master');
-        await this._baseApis.setAccountData(
-            `m.secret_storage.key.${keyId}`, keyInfo,
-        );
-    }
-
    /**
     * Get the key information for a given ID.
     *
@@ -185,15 +155,32 @@ export class SecretStorage extends EventEmitter {
        return !!(await this.getKey(keyId));
    }

-    async keyNeedsUpgrade(keyId) {
-        const keyInfo = await this.getKey(keyId);
-        if (keyInfo && keyInfo[1].algorithm === SECRET_STORAGE_ALGORITHM_V1_CURVE25519) {
-            return true;
+    /**
+     * Check whether a key matches what we expect based on the key info
+     *
+     * @param {Uint8Array} key the key to check
+     * @param {object} info the key info
+     *
+     * @return {boolean} whether or not the key matches
+     */
+    async checkKey(key, info) {
+        if (info.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
+            if (info.mac) {
+                const {mac} = await SecretStorage._calculateKeyCheck(key, info.iv);
+                return info.mac.replace(/=+$/g, '') === mac.replace(/=+$/g, '');
+            } else {
+                // if we have no information, we have to assume the key is right
+                return true;
+            }
        } else {
-            return false;
+            throw new Error("Unknown algorithm");
        }
    }

+    static async _calculateKeyCheck(key, iv) {
+        return await encryptAES(ZERO_STR, key, "", iv);
+    }
+
    /**
     * Store an encrypted secret on the server
     *
@@ -227,15 +214,11 @@ export class SecretStorage extends EventEmitter {
            }

            // encrypt secret, based on the algorithm
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
-            {
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                const keys = {[keyId]: keyInfo};
                const [, encryption] = await this._getSecretStorageKey(keys, name);
                encrypted[keyId] = await encryption.encrypt(secret);
-                break;
-            }
-            default:
+            } else {
                logger.warn("unknown algorithm for secret storage key " + keyId
                            + ": " + keyInfo.algorithm);
                // do nothing if we don't understand the encryption algorithm
@@ -301,27 +284,19 @@ export class SecretStorage extends EventEmitter {
                "m.secret_storage.key." + keyId,
            );
            const encInfo = secretInfo.encrypted[keyId];
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
                    keys[keyId] = keyInfo;
                }
-                break;
-            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-                if (
-                    keyInfo.pubkey && (
-                        (encInfo.ciphertext && encInfo.mac && encInfo.ephemeral) ||
-                        encInfo.passthrough
-                    )
-                ) {
-                    keys[keyId] = keyInfo;
-                }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
            }
        }

+        if (Object.keys(keys).length === 0) {
+            throw new Error(`Could not decrypt ${name} because none of ` +
+                `the keys it is encrypted with are for a supported algorithm`);
+        }
+
        let keyId;
        let decryption;
        try {
@@ -331,8 +306,9 @@ export class SecretStorage extends EventEmitter {
            const encInfo = secretInfo.encrypted[keyId];

            // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) return decryption.get_private_key();
+            // since we just want to return the key itself. It must be base64
+            // encoded, since this is how a key would normally be stored.
+            if (encInfo.passthrough) return encodeBase64(decryption.get_private_key());

            return await decryption.decrypt(encInfo);
        } finally {
@@ -366,8 +342,7 @@ export class SecretStorage extends EventEmitter {

        const ret = {};

-        // check if secret is encrypted by a known/trusted secret and
-        // encryption looks sane
+        // filter secret encryption keys with supported algorithm
        for (const keyId of Object.keys(secretInfo.encrypted)) {
            // get key information from key storage
            const keyInfo = await this._baseApis.getAccountDataFromServer(
@@ -376,49 +351,11 @@ export class SecretStorage extends EventEmitter {
            if (!keyInfo) continue;
            const encInfo = secretInfo.encrypted[keyId];

-            // We don't actually need the decryption object if it's a passthrough
-            // since we just want to return the key itself.
-            if (encInfo.passthrough) {
-                try {
-                    pkVerify(
-                        keyInfo,
-                        this._crossSigningInfo.getId('master'),
-                        this._crossSigningInfo.userId,
-                    );
-                } catch (e) {
-                    // not trusted, so move on to the next key
-                    continue;
-                }
-                ret[keyId] = keyInfo;
-                continue;
-            }
-
-            switch (keyInfo.algorithm) {
-            case SECRET_STORAGE_ALGORITHM_V1_AES:
+            // only use keys we understand the encryption algorithm of
+            if (keyInfo.algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
                if (encInfo.iv && encInfo.ciphertext && encInfo.mac) {
                    ret[keyId] = keyInfo;
                }
-                break;
-            case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-                if (keyInfo.pubkey && encInfo.ciphertext && encInfo.mac
-                    && encInfo.ephemeral) {
-                    if (checkKey) {
-                        try {
-                            pkVerify(
-                                keyInfo,
-                                this._crossSigningInfo.getId('master'),
-                                this._crossSigningInfo.userId,
-                            );
-                        } catch (e) {
-                            // not trusted, so move on to the next key
-                            continue;
-                        }
-                    }
-                    ret[keyId] = keyInfo;
-                }
-                break;
-            default:
-                // do nothing if we don't understand the encryption algorithm
            }
        }
        return Object.keys(ret).length ? ret : null;
@@ -436,6 +373,7 @@ export class SecretStorage extends EventEmitter {
        const requestId = this._baseApis.makeTxnId();

        const requestControl = this._requests[requestId] = {
+            name,
            devices,
        };
        const promise = new Promise((resolve, reject) => {
@@ -545,7 +483,7 @@ export class SecretStorage extends EventEmitter {
                    this._baseApis,
                    {
                        [sender]: [
-                            await this._baseApis.getStoredDevice(sender, deviceId),
+                            this._baseApis.getStoredDevice(sender, deviceId),
                        ],
                    },
                );
@@ -555,7 +493,7 @@ export class SecretStorage extends EventEmitter {
                    this._baseApis.deviceId,
                    this._baseApis._crypto._olmDevice,
                    sender,
-                    this._baseApis._crypto.getStoredDevice(sender, deviceId),
+                    this._baseApis.getStoredDevice(sender, deviceId),
                    payload,
                );
                const contentMap = {
@@ -599,6 +537,10 @@ export class SecretStorage extends EventEmitter {
                return;
            }

+            logger.log(
+                `Successfully received secret ${requestControl.name} ` +
+                `from ${deviceInfo.deviceId}`,
+            );
            requestControl.resolve(content.secret);
        }
    }
@@ -622,9 +564,7 @@ export class SecretStorage extends EventEmitter {
            throw new Error("App returned unknown key from getSecretStorageKey!");
        }

-        switch (keys[keyId].algorithm) {
-        case SECRET_STORAGE_ALGORITHM_V1_AES:
-        {
+        if (keys[keyId].algorithm === SECRET_STORAGE_ALGORITHM_V1_AES) {
            const decryption = {
                encrypt: async function(secret) {
                    return await encryptAES(secret, privateKey, name);
@@ -634,36 +574,7 @@ export class SecretStorage extends EventEmitter {
                },
            };
            return [keyId, decryption];
-        }
-        case SECRET_STORAGE_ALGORITHM_V1_CURVE25519:
-        {
-            const pkDecryption = new global.Olm.PkDecryption();
-            let pubkey;
-            try {
-                pubkey = pkDecryption.init_with_private_key(privateKey);
-            } catch (e) {
-                pkDecryption.free();
-                throw new Error("getSecretStorageKey callback returned invalid key");
-            }
-            if (pubkey !== keys[keyId].pubkey) {
-                pkDecryption.free();
-                throw new Error(
-                    "getSecretStorageKey callback returned incorrect key",
-                );
-            }
-            const decryption = {
-                free: pkDecryption.free.bind(pkDecryption),
-                decrypt: async function(encInfo) {
-                    return pkDecryption.decrypt(
-                        encInfo.ephemeral, encInfo.mac, encInfo.ciphertext,
-                    );
-                },
-                // needed for passthrough
-                get_private_key: pkDecryption.get_private_key.bind(pkDecryption),
-            };
-            return [keyId, decryption];
-        }
-        default:
+        } else {
            throw new Error("Unknown key type: " + keys[keyId].algorithm);
        }
    }
@@ -29,14 +29,20 @@ const zerosalt = new Uint8Array(8);
 * @param {string} data the plaintext to encrypt
 * @param {Uint8Array} key the encryption key to use
 * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
 */
-async function encryptNode(data, key, name) {
+async function encryptNode(data, key, name, ivStr) {
    const crypto = getCrypto();
    if (!crypto) {
        throw new Error("No usable crypto implementation");
    }

-    const iv = crypto.randomBytes(16);
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = crypto.randomBytes(16);
+    }

    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
    // (which would mean we wouldn't be able to decrypt on Android). The loss
@@ -78,9 +84,9 @@ async function decryptNode(data, key, name) {
    const [aesKey, hmacKey] = deriveKeysNode(key, name);

    const hmac = crypto.createHmac("sha256", hmacKey)
-        .update(data.ciphertext, "base64").digest("base64");
+        .update(data.ciphertext, "base64").digest("base64").replace(/=+$/g, '');

-    if (hmac !== data.mac) {
+    if (hmac !== data.mac.replace(/=+$/g, '')) {
        throw new Error(`Error decrypting secret ${name}: bad MAC`);
    }

@@ -112,10 +118,16 @@ function deriveKeysNode(key, name) {
 * @param {string} data the plaintext to encrypt
 * @param {Uint8Array} key the encryption key to use
 * @param {string} name the name of the secret
+ * @param {string} ivStr the initialization vector to use
 */
-async function encryptBrowser(data, key, name) {
-    const iv = new Uint8Array(16);
-    window.crypto.getRandomValues(iv);
+async function encryptBrowser(data, key, name, ivStr) {
+    let iv;
+    if (ivStr) {
+        iv = decodeBase64(ivStr);
+    } else {
+        iv = new Uint8Array(16);
+        window.crypto.getRandomValues(iv);
+    }

    // clear bit 63 of the IV to stop us hitting the 64-bit counter boundary
    // (which would mean we wouldn't be able to decrypt on Android). The loss
@@ -311,7 +311,7 @@ MegolmEncryption.prototype._ensureOutboundSession = async function(
                        }

                        await this._shareKeyWithDevices(
-                            session, key, payload, retryDevices, failedDevices,
+                            session, key, payload, retryDevices, failedDevices, 30000,
                        );

                        await this._notifyFailedOlmDevices(session, key, failedDevices);
@@ -521,6 +521,33 @@ MegolmEncryption.prototype._encryptAndSendKeysToDevices = function(
    }

    return Promise.all(promises).then(() => {
+        // prune out any devices that encryptMessageForDevice could not encrypt for,
+        // in which case it will have just not added anything to the ciphertext object.
+        // There's no point sending messages to devices if we couldn't encrypt to them,
+        // since that's effectively a blank message.
+        for (const userId of Object.keys(contentMap)) {
+            for (const deviceId of Object.keys(contentMap[userId])) {
+                if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) {
+                    logger.log(
+                        "No ciphertext for device " +
+                        userId + ":" + deviceId + ": pruning",
+                    );
+                    delete contentMap[userId][deviceId];
+                }
+            }
+            // No devices left for that user? Strip that too.
+            if (Object.keys(contentMap[userId]).length === 0) {
+                logger.log("Pruned all devices for user " + userId);
+                delete contentMap[userId];
+            }
+        }
+
+        // Is there anything left?
+        if (Object.keys(contentMap).length === 0) {
+            logger.log("No users left to send to: aborting");
+            return;
+        }
+
        return this._baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => {
            // store that we successfully uploaded the keys of the current slice
            for (const userId of Object.keys(contentMap)) {
@@ -988,7 +1015,7 @@ MegolmEncryption.prototype._getDevicesInRoom = async function(room) {
    // with them, which means that they will have announced any new devices via
    // device_lists in their /sync response.  This cache should then be maintained
    // using all the device_lists changes and left fields.
-    // See https://github.com/vector-im/riot-web/issues/2305 for details.
+    // See https://github.com/vector-im/element-web/issues/2305 for details.
    const devices = await this._crypto.downloadKeys(roomMembers, false);
    const blocked = {};
    // remove any blocked devices
@@ -1082,7 +1109,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
    //
    // then, if the key turns up while decryption is in progress (and
    // decryption fails), we will schedule a retry.
-    // (fixes https://github.com/vector-im/riot-web/issues/5001)
+    // (fixes https://github.com/vector-im/element-web/issues/5001)
    this._addEventToPendingList(event);

    let res;
@@ -1174,6 +1201,7 @@ MegolmDecryption.prototype.decryptEvent = async function(event) {
        senderCurve25519Key: res.senderKey,
        claimedEd25519Key: res.keysClaimed.ed25519,
        forwardingCurve25519KeyChain: res.forwardingCurve25519KeyChain,
+        untrusted: res.untrusted,
    };
 };

@@ -1296,7 +1324,6 @@ MegolmDecryption.prototype.onRoomKeyEvent = function(event) {
        keysClaimed = event.getKeysClaimed();
    }

-    logger.log(`Received and adding key for megolm session ${senderKey}|${sessionId}`);
    return this._olmDevice.addInboundGroupSession(
        content.room_id, senderKey, forwardingKeyChain, sessionId,
        content.session_key, keysClaimed,
@@ -1522,8 +1549,11 @@ MegolmDecryption.prototype._buildKeyForwardingMessage = async function(
 * @inheritdoc
 *
 * @param {module:crypto/OlmDevice.MegolmSessionData} session
+ * @param {object} [opts={}] options for the import
+ * @param {boolean} [opts.untrusted] whether the key should be considered as untrusted
+ * @param {string} [opts.source] where the key came from
 */
-MegolmDecryption.prototype.importRoomKey = function(session) {
+MegolmDecryption.prototype.importRoomKey = function(session, opts = {}) {
    return this._olmDevice.addInboundGroupSession(
        session.room_id,
        session.sender_key,
@@ -1532,8 +1562,9 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
        session.session_key,
        session.sender_claimed_keys,
        true,
+        opts.untrusted ? { untrusted: opts.untrusted } : {},
    ).then(() => {
-        if (this._crypto.backupInfo) {
+        if (this._crypto.backupInfo && opts.source !== "backup") {
            // don't wait for it to complete
            this._crypto.backupGroupSession(
                session.room_id,
@@ -1555,7 +1586,8 @@ MegolmDecryption.prototype.importRoomKey = function(session) {
 };

 /**
- * Have another go at decrypting events after we receive a key
+ * Have another go at decrypting events after we receive a key. Resolves once
+ * decryption has been re-attempted on all events.
 *
 * @private
 * @param {String} senderKey
@@ -1574,21 +1606,17 @@ MegolmDecryption.prototype._retryDecryption = async function(senderKey, sessionI
        return true;
    }

-    pending.delete(sessionId);
-    if (pending.size === 0) {
-        this._pendingEvents[senderKey];
-    }
+    logger.debug("Retrying decryption on events", [...pending]);

    await Promise.all([...pending].map(async (ev) => {
        try {
-            await ev.attemptDecryption(this._crypto);
+            await ev.attemptDecryption(this._crypto, true);
        } catch (e) {
            // don't die if something goes wrong
        }
    }));

-    // ev.attemptDecryption will re-add to this._pendingEvents if an event
-    // couldn't be decrypted
+    // If decrypted successfully, they'll have been removed from _pendingEvents
    return !((this._pendingEvents[senderKey] || {})[sessionId]);
 };

@@ -264,6 +264,25 @@ OlmDecryption.prototype.decryptEvent = async function(event) {
 */
 OlmDecryption.prototype._decryptMessage = async function(
    theirDeviceIdentityKey, message,
+) {
+    // This is a wrapper that serialises decryptions of prekey messages, because
+    // otherwise we race between deciding we have no active sessions for the message
+    // and creating a new one, which we can only do once because it removes the OTK.
+    if (message.type !== 0) {
+        // not a prekey message: we can safely just try & decrypt it
+        return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+    } else {
+        const myPromise = this._olmDevice._olmPrekeyPromise.then(() => {
+            return this._reallyDecryptMessage(theirDeviceIdentityKey, message);
+        });
+        // we want the error, but don't propagate it to the next decryption
+        this._olmDevice._olmPrekeyPromise = myPromise.catch(() => {});
+        return await myPromise;
+    }
+};
+
+OlmDecryption.prototype._reallyDecryptMessage = async function(
+    theirDeviceIdentityKey, message,
 ) {
    const sessionIds = await this._olmDevice.getSessionIdsForDevice(
        theirDeviceIdentityKey,
@@ -207,6 +207,25 @@ export async function ensureOlmSessionsForDevices(
        for (const deviceInfo of devices) {
            const deviceId = deviceInfo.deviceId;
            const key = deviceInfo.getIdentityKey();
+
+            if (key === olmDevice.deviceCurve25519Key) {
+                // We should never be trying to start a session with ourself.
+                // Apart from talking to yourself being the first sign of madness,
+                // olm sessions can't do this because they get confused when
+                // they get a message and see that the 'other side' has started a
+                // new chain when this side has an active sender chain.
+                // If you see this message being logged in the wild, we should find
+                // the thing that is trying to send Olm messages to itself and fix it.
+                logger.info("Attempted to start session with ourself! Ignoring");
+                // We must fill in the section in the return value though, as callers
+                // expect it to be there.
+                result[userId][deviceId] = {
+                    device: deviceInfo,
+                    sessionId: null,
+                };
+                continue;
+            }
+
            if (!olmDevice._sessionsInProgress[key]) {
                // pre-emptively mark the session as in-progress to avoid race
                // conditions.  If we find that we already have a session, then
@@ -238,6 +257,11 @@ export async function ensureOlmSessionsForDevices(
                delete resolveSession[key];
            }
            if (sessionId === null || force) {
+                if (force) {
+                    logger.info("Forcing new Olm session for " + userId + ":" + deviceId);
+                } else {
+                    logger.info("Making new Olm session for " + userId + ":" + deviceId);
+                }
                devicesWithoutSession.push([userId, deviceId]);
            }
            result[userId][deviceId] = {
@@ -277,6 +301,14 @@ export async function ensureOlmSessionsForDevices(
            const deviceInfo = devices[j];
            const deviceId = deviceInfo.deviceId;
            const key = deviceInfo.getIdentityKey();
+
+            if (key === olmDevice.deviceCurve25519Key) {
+                // We've already logged about this above. Skip here too
+                // otherwise we'll log saying there are no one-time keys
+                // which will be confusing.
+                continue;
+            }
+
            if (result[userId][deviceId].sessionId && !force) {
                // we already have a result for this device
                continue;
@@ -59,8 +59,8 @@ export function decodeRecoveryKey(recoverykey) {
        throw new Error("Incorrect length");
    }

-    return result.slice(
+    return Uint8Array.from(result.slice(
        OLM_RECOVERY_KEY_PREFIX.length,
        OLM_RECOVERY_KEY_PREFIX.length + global.Olm.PRIVATE_KEY_LENGTH,
-    );
+    ));
 }
@@ -131,7 +131,7 @@ export class Backend {

        cursorReq.onsuccess = (ev) => {
            const cursor = ev.target.result;
-            if(!cursor) {
+            if (!cursor) {
                // no match found
                callback(null);
                return;
@@ -203,6 +203,23 @@ export class Backend {
        return promiseifyTxn(txn).then(() => result);
    }

+    /**
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} All elements in a given state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return new Promise((resolve, reject) => {
+            const txn = this._db.transaction("outgoingRoomKeyRequests", "readonly");
+            const store = txn.objectStore("outgoingRoomKeyRequests");
+            const index = store.index("state");
+            const request = index.getAll(wantedState);
+
+            request.onsuccess = (ev) => resolve(ev.target.result);
+            request.onerror = (ev) => reject(ev.target.error);
+        });
+    }
+
    getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
        let stateIndex = 0;
        const results = [];
@@ -225,6 +225,17 @@ export class IndexedDBCryptoStore {
        return this._backend.getOutgoingRoomKeyRequestByState(wantedStates);
    }

+    /**
+     * Look for room key requests by state –
+     * unlike above, return a list of all entries in one state.
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} Returns an array of requests in the given state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return this._backend.getAllOutgoingRoomKeyRequestsByState(wantedState);
+    }
+
    /**
     * Look for room key requests by target device and state
     *
@@ -369,7 +369,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {

    getSecretStorePrivateKey(txn, func, type) {
        const key = getJsonItem(this.store, E2E_PREFIX + `ssss_cache.${type}`);
-        func(key ? Uint8Array.from(key) : key);
+        func(key);
    }

    storeCrossSigningKeys(txn, keys) {
@@ -380,7 +380,7 @@ export class LocalStorageCryptoStore extends MemoryCryptoStore {

    storeSecretStorePrivateKey(txn, type, key) {
        setJsonItem(
-            this.store, E2E_PREFIX + `ssss_cache.${type}`, Array.from(key),
+            this.store, E2E_PREFIX + `ssss_cache.${type}`, key,
        );
    }

@@ -166,6 +166,19 @@ export class MemoryCryptoStore {
        return Promise.resolve(null);
    }

+    /**
+     *
+     * @param {Number} wantedState
+     * @return {Promise<Array<*>>} All OutgoingRoomKeyRequests in state
+     */
+    getAllOutgoingRoomKeyRequestsByState(wantedState) {
+        return Promise.resolve(
+            this._outgoingRoomKeyRequests.filter(
+                (r) => r.state == wantedState,
+            ),
+        );
+    }
+
    getOutgoingRoomKeyRequestsByTarget(userId, deviceId, wantedStates) {
        const results = [];

@@ -1,5 +1,6 @@
 /*
 Copyright 2018 New Vector Ltd
+Copyright 2020 The Matrix.org Foundation C.I.C.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -24,8 +25,7 @@ import {EventEmitter} from 'events';
 import {logger} from '../../logger';
 import {DeviceInfo} from '../deviceinfo';
 import {newTimeoutError} from "./Error";
-import {CrossSigningInfo} from "../CrossSigning";
-import {decodeBase64} from "../olmlib";
+import {requestKeysDuringVerification} from "../CrossSigning";

 const timeoutException = new Error("Verification timed out");

@@ -78,8 +78,6 @@ export class VerificationBase extends EventEmitter {
        this._transactionTimeoutTimer = null;
    }

-    static keyRequestTimeoutMs = 1000 * 60;
-
    get initiatedByMe() {
        // if there is no start event yet,
        // we probably want to send it,
@@ -121,6 +119,11 @@ export class VerificationBase extends EventEmitter {
        if (this._done) {
            return Promise.reject(new Error("Verification is already done"));
        }
+        const existingEvent = this.request.getEventFromOtherParty(type);
+        if (existingEvent) {
+            return Promise.resolve(existingEvent);
+        }
+
        this._expectedEvent = type;
        return new Promise((resolve, reject) => {
            this._resolveEvent = resolve;
@@ -192,63 +195,7 @@ export class VerificationBase extends EventEmitter {
        if (!this._done) {
            this.request.onVerifierFinished();
            this._resolve();
-
-            //#region Cross-signing keys request
-            // If this is a self-verification, ask the other party for keys
-            if (this._baseApis.getUserId() !== this.userId) {
-                return;
-            }
-            console.log("VerificationBase.done: Self-verification done; requesting keys");
-            /* This happens asynchronously, and we're not concerned about
-             * waiting for it.  We return here in order to test. */
-            return new Promise((resolve, reject) => {
-                const client = this._baseApis;
-                const original = client._crypto._crossSigningInfo;
-                const storage = client._crypto._secretStorage;
-
-                /* We already have all of the infrastructure we need to validate and
-                 * cache cross-signing keys, so instead of replicating that, here we
-                 * set up callbacks that request them from the other device and call
-                 * CrossSigningInfo.getCrossSigningKey() to validate/cache */
-                const crossSigning = new CrossSigningInfo(
-                    original.userId,
-                    { getCrossSigningKey: async (type) => {
-                        console.debug("VerificationBase.done: requesting secret",
-                                      type, this.deviceId);
-                        const { promise } =
-                            storage.request(`m.cross_signing.${type}`, [this.deviceId]);
-                        const result = await promise;
-                        const decoded = decodeBase64(result);
-                        return Uint8Array.from(decoded);
-                    } },
-                    original._cacheCallbacks,
-                );
-                crossSigning.keys = original.keys;
-
-                // XXX: get all keys out if we get one key out
-                // https://github.com/vector-im/riot-web/issues/12604
-                // then change here to reject on the timeout
-                /* Requests can be ignored, so don't wait around forever */
-                const timeout = new Promise((resolve, reject) => {
-                    setTimeout(
-                        resolve,
-                        VerificationBase.keyRequestTimeoutMs,
-                        new Error("Timeout"),
-                    );
-                });
-
-                /* We call getCrossSigningKey() for its side-effects */
-                return Promise.race([
-                    Promise.all([
-                        crossSigning.getCrossSigningKey("self_signing"),
-                        crossSigning.getCrossSigningKey("user_signing"),
-                    ]),
-                    timeout,
-                ]).then(resolve, reject);
-            }).catch((e) => {
-                console.warn("VerificationBase: failure while requesting keys:", e);
-            });
-            //#endregion
+            return requestKeysDuringVerification(this._baseApis, this.userId, this.deviceId);
        }
    }

@@ -256,6 +203,7 @@ export class VerificationBase extends EventEmitter {
        this._endTimer(); // always kill the activity timer
        if (!this._done) {
            this.cancelled = true;
+            this.request.onVerifierCancelled();
            if (this.userId && this.deviceId) {
                // send a cancellation to the other user (if it wasn't
                // cancelled by the other user)
@@ -338,7 +286,7 @@ export class VerificationBase extends EventEmitter {

        for (const [keyId, keyInfo] of Object.entries(keys)) {
            const deviceId = keyId.split(':', 2)[1];
-            const device = await this._baseApis.getStoredDevice(userId, deviceId);
+            const device = this._baseApis.getStoredDevice(userId, deviceId);
            if (device) {
                await verifier(keyId, device, keyInfo);
                verifiedDevices.push(deviceId);
@@ -23,8 +23,9 @@ limitations under the License.
 import {VerificationBase as Base} from "./Base";
 import {
    newKeyMismatchError,
-    newUserMismatchError,
+    newUserCancelledError,
 } from './Error';
+import {encodeUnpaddedBase64, decodeBase64} from "../olmlib";

 export const SHOW_QR_CODE_METHOD = "m.qr_code.show.v1";
 export const SCAN_QR_CODE_METHOD = "m.qr_code.scan.v1";
@@ -49,47 +50,46 @@ export class ReciprocateQRCode extends Base {
                "with this method yet.");
        }

-        const targetUserId = this.startEvent.getSender();
-        if (!this.userId) {
-            console.log("Asking to confirm user ID");
-            this.userId = await new Promise((resolve, reject) => {
-                this.emit("confirm_user_id", {
-                    userId: targetUserId,
-                    confirm: resolve, // takes a userId
-                    cancel: () => reject(newUserMismatchError()),
-                });
-            });
-        } else if (targetUserId !== this.userId) {
-            throw newUserMismatchError({
-                expected: this.userId,
-                actual: targetUserId,
-            });
-        }
-
-        if (this.startEvent.getContent()['secret'] !== this.request.encodedSharedSecret) {
+        const {qrCodeData} = this.request;
+        // 1. check the secret
+        if (this.startEvent.getContent()['secret'] !== qrCodeData.encodedSharedSecret) {
            throw newKeyMismatchError();
        }

-        // If we've gotten this far, verify the user's master cross signing key
-        const xsignInfo = this._baseApis.getStoredCrossSigningForUser(this.userId);
-        if (!xsignInfo) throw new Error("Missing cross signing info");
-
-        const masterKey = xsignInfo.getId("master");
-        const masterKeyId = `ed25519:${masterKey}`;
-        const keys = {[masterKeyId]: masterKey};
-
-        const devices = (await this._baseApis.getStoredDevicesForUser(this.userId)) || [];
-        const targetDevice = devices.find(d => {
-            return d.deviceId === this.request.targetDevice.deviceId;
+        // 2. ask if other user shows shield as well
+        await new Promise((resolve, reject) => {
+            this.reciprocateQREvent = {
+                confirm: resolve,
+                cancel: () => reject(newUserCancelledError()),
+            };
+            this.emit("show_reciprocate_qr", this.reciprocateQREvent);
        });
-        if (!targetDevice) throw new Error("Device not found, somehow");
-        keys[`ed25519:${targetDevice.deviceId}`] = targetDevice.getFingerprint();

-        if (this.request.requestingUserId === this.request.receivingUserId) {
-            delete keys[masterKeyId];
+        // 3. determine key to sign / mark as trusted
+        const keys = {};
+
+        switch (qrCodeData.mode) {
+            case MODE_VERIFY_OTHER_USER: {
+                // add master key to keys to be signed, only if we're not doing self-verification
+                const masterKey = qrCodeData.otherUserMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_TRUSTED: {
+                const deviceId = this.request.targetDevice.deviceId;
+                keys[`ed25519:${deviceId}`] = qrCodeData.otherDeviceKey;
+                break;
+            }
+            case MODE_VERIFY_SELF_UNTRUSTED: {
+                const masterKey = qrCodeData.myMasterKey;
+                keys[`ed25519:${masterKey}`] = masterKey;
+                break;
+            }
        }

+        // 4. sign the key (or mark own MSK as verified in case of MODE_VERIFY_SELF_TRUSTED)
        await this._verifyKeys(this.userId, keys, (keyId, device, keyInfo) => {
+            // make sure the device has the expected keys
            const targetKey = keys[keyId];
            if (!targetKey) throw newKeyMismatchError();

@@ -106,8 +106,200 @@ export class ReciprocateQRCode extends Base {
                    throw newKeyMismatchError();
                }
            }
-
-            // Otherwise it is probably fine
        });
    }
 }
+
+const CODE_VERSION = 0x02; // the version of binary QR codes we support
+const BINARY_PREFIX = "MATRIX"; // ASCII, used to prefix the binary format
+const MODE_VERIFY_OTHER_USER = 0x00; // Verifying someone who isn't us
+const MODE_VERIFY_SELF_TRUSTED = 0x01; // We trust the master key
+const MODE_VERIFY_SELF_UNTRUSTED = 0x02; // We do not trust the master key
+
+export class QRCodeData {
+    constructor(
+        mode, sharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey, buffer,
+    ) {
+        this._sharedSecret = sharedSecret;
+        this._mode = mode;
+        this._otherUserMasterKey = otherUserMasterKey;
+        this._otherDeviceKey = otherDeviceKey;
+        this._myMasterKey = myMasterKey;
+        this._buffer = buffer;
+    }
+
+    static async create(request, client) {
+        const sharedSecret = QRCodeData._generateSharedSecret();
+        const mode = QRCodeData._determineMode(request, client);
+        let otherUserMasterKey = null;
+        let otherDeviceKey = null;
+        let myMasterKey = null;
+        if (mode === MODE_VERIFY_OTHER_USER) {
+            const otherUserCrossSigningInfo =
+                client.getStoredCrossSigningForUser(request.otherUserId);
+            otherUserMasterKey = otherUserCrossSigningInfo.getId("master");
+        } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
+            otherDeviceKey = await QRCodeData._getOtherDeviceKey(request, client);
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            const myUserId = client.getUserId();
+            const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+            myMasterKey = myCrossSigningInfo.getId("master");
+        }
+        const qrData = QRCodeData._generateQrData(
+            request, client, mode,
+            sharedSecret,
+            otherUserMasterKey,
+            otherDeviceKey,
+            myMasterKey,
+        );
+        const buffer = QRCodeData._generateBuffer(qrData);
+        return new QRCodeData(mode, sharedSecret,
+            otherUserMasterKey, otherDeviceKey, myMasterKey, buffer);
+    }
+
+    get buffer() {
+        return this._buffer;
+    }
+
+    get mode() {
+        return this._mode;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_SELF_TRUSTED
+     * @return {string} device key of other party at time of generating QR code
+     */
+    get otherDeviceKey() {
+        return this._otherDeviceKey;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_OTHER_USER
+     * @return {string} master key of other party at time of generating QR code
+     */
+    get otherUserMasterKey() {
+        return this._otherUserMasterKey;
+    }
+
+    /**
+     * only set when mode is MODE_VERIFY_SELF_UNTRUSTED
+     * @return {string} own master key at time of generating QR code
+     */
+    get myMasterKey() {
+        return this._myMasterKey;
+    }
+
+    /**
+     * The unpadded base64 encoded shared secret.
+     */
+    get encodedSharedSecret() {
+        return this._sharedSecret;
+    }
+
+    static _generateSharedSecret() {
+        const secretBytes = new Uint8Array(11);
+        global.crypto.getRandomValues(secretBytes);
+        return encodeUnpaddedBase64(secretBytes);
+    }
+
+    static async _getOtherDeviceKey(request, client) {
+        const myUserId = client.getUserId();
+        const otherDevice = request.targetDevice;
+        const otherDeviceId = otherDevice ? otherDevice.deviceId : null;
+        const device = client.getStoredDevice(myUserId, otherDeviceId);
+        if (!device) {
+            throw new Error("could not find device " + otherDeviceId);
+        }
+        const key = device.getFingerprint();
+        return key;
+    }
+
+    static _determineMode(request, client) {
+        const myUserId = client.getUserId();
+        const otherUserId = request.otherUserId;
+
+        let mode = MODE_VERIFY_OTHER_USER;
+        if (myUserId === otherUserId) {
+            // Mode changes depending on whether or not we trust the master cross signing key
+            const myTrust = client.checkUserTrust(myUserId);
+            if (myTrust.isCrossSigningVerified()) {
+                mode = MODE_VERIFY_SELF_TRUSTED;
+            } else {
+                mode = MODE_VERIFY_SELF_UNTRUSTED;
+            }
+        }
+        return mode;
+    }
+
+    static _generateQrData(request, client, mode,
+        encodedSharedSecret, otherUserMasterKey,
+        otherDeviceKey, myMasterKey,
+    ) {
+        const myUserId = client.getUserId();
+        const transactionId = request.channel.transactionId;
+        const qrData = {
+            prefix: BINARY_PREFIX,
+            version: CODE_VERSION,
+            mode,
+            transactionId,
+            firstKeyB64: '', // worked out shortly
+            secondKeyB64: '', // worked out shortly
+            secretB64: encodedSharedSecret,
+        };
+
+        const myCrossSigningInfo = client.getStoredCrossSigningForUser(myUserId);
+
+        if (mode === MODE_VERIFY_OTHER_USER) {
+            // First key is our master cross signing key
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
+            // Second key is the other user's master cross signing key
+            qrData.secondKeyB64 = otherUserMasterKey;
+        } else if (mode === MODE_VERIFY_SELF_TRUSTED) {
+            // First key is our master cross signing key
+            qrData.firstKeyB64 = myCrossSigningInfo.getId("master");
+            qrData.secondKeyB64 = otherDeviceKey;
+        } else if (mode === MODE_VERIFY_SELF_UNTRUSTED) {
+            // First key is our device's key
+            qrData.firstKeyB64 = client.getDeviceEd25519Key();
+            // Second key is what we think our master cross signing key is
+            qrData.secondKeyB64 = myMasterKey;
+        }
+        return qrData;
+    }
+
+    static _generateBuffer(qrData) {
+        let buf = Buffer.alloc(0); // we'll concat our way through life
+
+        const appendByte = (b) => {
+            const tmpBuf = Buffer.from([b]);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendInt = (i) => {
+            const tmpBuf = Buffer.alloc(2);
+            tmpBuf.writeInt16BE(i, 0);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendStr = (s, enc, withLengthPrefix = true) => {
+            const tmpBuf = Buffer.from(s, enc);
+            if (withLengthPrefix) appendInt(tmpBuf.byteLength);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+        const appendEncBase64 = (b64) => {
+            const b = decodeBase64(b64);
+            const tmpBuf = Buffer.from(b);
+            buf = Buffer.concat([buf, tmpBuf]);
+        };
+
+        // Actually build the buffer for the QR code
+        appendStr(qrData.prefix, "ascii", false);
+        appendByte(qrData.version);
+        appendByte(qrData.mode);
+        appendStr(qrData.transactionId, "utf-8");
+        appendEncBase64(qrData.firstKeyB64);
+        appendEncBase64(qrData.secondKeyB64);
+        appendEncBase64(qrData.secretB64);
+
+        return buf;
+    }
+}
@@ -175,11 +175,33 @@ function calculateMAC(olmSAS, method) {
    };
 }

+const calculateKeyAgreement = {
+    "curve25519-hkdf-sha256": function(sas, olmSAS, bytes) {
+        const ourInfo = `${sas._baseApis.getUserId()}|${sas._baseApis.deviceId}|`
+              + `${sas.ourSASPubKey}|`;
+        const theirInfo = `${sas.userId}|${sas.deviceId}|${sas.theirSASPubKey}|`;
+        const sasInfo =
+            "MATRIX_KEY_VERIFICATION_SAS|"
+              + (sas.initiatedByMe ? ourInfo + theirInfo : theirInfo + ourInfo)
+              + sas._channel.transactionId;
+        return olmSAS.generate_bytes(sasInfo, bytes);
+    },
+    "curve25519": function(sas, olmSAS, bytes) {
+        const ourInfo = `${sas._baseApis.getUserId()}${sas._baseApis.deviceId}`;
+        const theirInfo = `${sas.userId}${sas.deviceId}`;
+        const sasInfo =
+            "MATRIX_KEY_VERIFICATION_SAS"
+              + (sas.initiatedByMe ? ourInfo + theirInfo : theirInfo + ourInfo)
+              + sas._channel.transactionId;
+        return olmSAS.generate_bytes(sasInfo, bytes);
+    },
+};
+
 /* lists of algorithms/methods that are supported.  The key agreement, hashes,
 * and MAC lists should be sorted in order of preference (most preferred
 * first).
 */
-const KEY_AGREEMENT_LIST = ["curve25519"];
+const KEY_AGREEMENT_LIST = ["curve25519-hkdf-sha256", "curve25519"];
 const HASHES_LIST = ["sha256"];
 const MAC_LIST = ["hkdf-hmac-sha256", "hmac-sha256"];
 const SAS_LIST = Object.keys(sasGenerators);
@@ -291,12 +313,14 @@ export class SAS extends Base {
        if (typeof content.commitment !== "string") {
            throw newInvalidMessageError();
        }
+        const keyAgreement = content.key_agreement_protocol;
        const macMethod = content.message_authentication_code;
        const hashCommitment = content.commitment;
        const olmSAS = new global.Olm.SAS();
        try {
-            this._send("m.key.verification.key", {
-                key: olmSAS.get_pubkey(),
+            this.ourSASPubKey = olmSAS.get_pubkey();
+            await this._send("m.key.verification.key", {
+                key: this.ourSASPubKey,
            });


@@ -308,19 +332,20 @@ export class SAS extends Base {
            if (olmutil.sha256(commitmentStr) !== hashCommitment) {
                throw newMismatchedCommitmentError();
            }
+            this.theirSASPubKey = content.key;
            olmSAS.set_their_key(content.key);

-            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
-                  + this._baseApis.getUserId() + this._baseApis.deviceId
-                  + this.userId + this.deviceId
-                  + this._channel.transactionId;
-            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6);
            const verifySAS = new Promise((resolve, reject) => {
                this.sasEvent = {
                    sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch (err) {
+                            reject(err);
+                        }
                    },
                    cancel: () => reject(newUserCancelledError()),
                    mismatch: () => reject(newMismatchedSASError()),
@@ -377,7 +402,7 @@ export class SAS extends Base {
        const olmSAS = new global.Olm.SAS();
        try {
            const commitmentStr = olmSAS.get_pubkey() + anotherjson.stringify(content);
-            this._send("m.key.verification.accept", {
+            await this._send("m.key.verification.accept", {
                key_agreement_protocol: keyAgreement,
                hash: hashMethod,
                message_authentication_code: macMethod,
@@ -390,22 +415,24 @@ export class SAS extends Base {
            let e = await this._waitForEvent("m.key.verification.key");
            // FIXME: make sure event is properly formed
            content = e.getContent();
+            this.theirSASPubKey = content.key;
            olmSAS.set_their_key(content.key);
-            this._send("m.key.verification.key", {
-                key: olmSAS.get_pubkey(),
+            this.ourSASPubKey = olmSAS.get_pubkey();
+            await this._send("m.key.verification.key", {
+                key: this.ourSASPubKey,
            });

-            const sasInfo = "MATRIX_KEY_VERIFICATION_SAS"
-                  + this.userId + this.deviceId
-                  + this._baseApis.getUserId() + this._baseApis.deviceId
-                  + this._channel.transactionId;
-            const sasBytes = olmSAS.generate_bytes(sasInfo, 6);
+            const sasBytes = calculateKeyAgreement[keyAgreement](this, olmSAS, 6);
            const verifySAS = new Promise((resolve, reject) => {
                this.sasEvent = {
                    sas: generateSas(sasBytes, sasMethods),
-                    confirm: () => {
-                        this._sendMAC(olmSAS, macMethod);
-                        resolve();
+                    confirm: async () => {
+                        try {
+                            await this._sendMAC(olmSAS, macMethod);
+                            resolve();
+                        } catch (err) {
+                            reject(err);
+                        }
                    },
                    cancel: () => reject(newUserCancelledError()),
                    mismatch: () => reject(newMismatchedSASError()),
@@ -461,7 +488,7 @@ export class SAS extends Base {
            keyList.sort().join(","),
            baseInfo + "KEY_IDS",
        );
-        this._send("m.key.verification.mac", { mac, keys });
+        return this._send("m.key.verification.mac", { mac, keys });
    }

    async _checkMAC(olmSAS, content, method) {
@@ -44,11 +44,6 @@ export class InRoomChannel {
        this._requestEventId = null;
    }

-    /** Whether this channel needs m.key.verification.done messages to be sent after a successful verification */
-    get needsDoneMessage() {
-        return true;
-    }
-
    get receiveStartFromOtherDevices() {
        return true;
    }
@@ -183,6 +178,11 @@ export class InRoomChannel {
     * @returns {Promise} a promise that resolves when any requests as an anwser to the passed-in event are sent.
     */
    async handleEvent(event, request, isLiveEvent) {
+        // prevent processing the same event multiple times, as under
+        // some circumstances Room.timeline can get emitted twice for the same event
+        if (request.hasEventId(event.getId())) {
+            return;
+        }
        const type = InRoomChannel.getEventType(event);
        // do validations that need state (roomId, userId),
        // ignore if invalid
@@ -61,10 +61,6 @@ export class ToDeviceChannel {
        return this._deviceId;
    }

-    get needsDoneMessage() {
-        return false;
-    }
-
    static getEventType(event) {
        return event.getType();
    }
@@ -360,4 +356,12 @@ export class ToDeviceRequests {
            }
        }
    }
+
+    getRequestsInProgress(userId) {
+        const requestsByTxnId = this._requestsByUserId.get(userId);
+        if (requestsByTxnId) {
+            return Array.from(requestsByTxnId.values()).filter(r => r.pending);
+        }
+        return [];
+    }
 }
@@ -23,17 +23,19 @@ import {
    newUnexpectedMessageError,
    newUnknownMethodError,
 } from "../Error";
-import * as olmlib from "../../olmlib";
+import {QRCodeData, SCAN_QR_CODE_METHOD} from "../QRCode";
+
+// How long after the event's timestamp that the request times out
+const TIMEOUT_FROM_EVENT_TS = 10 * 60 * 1000; // 10 minutes
+
+// How long after we receive the event that the request times out
+const TIMEOUT_FROM_EVENT_RECEIPT = 2 * 60 * 1000; // 2 minutes

-// the recommended amount of time before a verification request
-// should be (automatically) cancelled without user interaction
-// and ignored.
-const VERIFICATION_REQUEST_TIMEOUT = 10 * 60 * 1000; //10m
 // to avoid almost expired verification notifications
 // from showing a notification and almost immediately
 // disappearing, also ignore verification requests that
 // are this amount of time away from expiring.
-const VERIFICATION_REQUEST_MARGIN = 3 * 1000; //3s
+const VERIFICATION_REQUEST_MARGIN = 3 * 1000; // 3 seconds


 export const EVENT_PREFIX = "m.key.verification.";
@@ -70,10 +72,19 @@ export class VerificationRequest extends EventEmitter {
        this._eventsByThem = new Map();
        this._observeOnly = false;
        this._timeoutTimer = null;
-        this._sharedSecret = null; // used for QR codes
        this._accepting = false;
        this._declining = false;
        this._verifierHasFinished = false;
+        this._cancelled = false;
+        this._chosenMethod = null;
+        // we keep a copy of the QR Code data (including other user master key) around
+        // for QR reciprocate verification, to protect against
+        // cross-signing identity reset between the .ready and .start event
+        // and signing the wrong key after .start
+        this._qrCodeData = null;
+
+        // The timestamp when we received the request event from the other side
+        this._requestReceivedAt = null;
    }

    /**
@@ -154,12 +165,31 @@ export class VerificationRequest extends EventEmitter {
        return this._commonMethods;
    }

+    /** the method picked in the .start event */
+    get chosenMethod() {
+        return this._chosenMethod;
+    }
+
+    calculateEventTimeout(event) {
+        let effectiveExpiresAt = this.channel.getTimestamp(event)
+            + TIMEOUT_FROM_EVENT_TS;
+
+        if (this._requestReceivedAt && !this.initiatedByMe &&
+            this.phase <= PHASE_REQUESTED
+        ) {
+            const expiresAtByReceipt = this._requestReceivedAt
+                + TIMEOUT_FROM_EVENT_RECEIPT;
+            effectiveExpiresAt = Math.min(effectiveExpiresAt, expiresAtByReceipt);
+        }
+
+        return Math.max(0, effectiveExpiresAt - Date.now());
+    }
+
    /** The current remaining amount of ms before the request should be automatically cancelled */
    get timeout() {
        const requestEvent = this._getEventByEither(REQUEST_TYPE);
        if (requestEvent) {
-            const elapsed = Date.now() - this.channel.getTimestamp(requestEvent);
-            return Math.max(0, VERIFICATION_REQUEST_TIMEOUT - elapsed);
+            return this.calculateEventTimeout(requestEvent);
        }
        return 0;
    }
@@ -201,14 +231,20 @@ export class VerificationRequest extends EventEmitter {
            this._phase !== PHASE_CANCELLED;
    }

+    /** Only set after a .ready if the other party can scan a QR code */
+    get qrCodeData() {
+        return this._qrCodeData;
+    }
+
    /** Checks whether the other party supports a given verification method.
     *  This is useful when setting up the QR code UI, as it is somewhat asymmetrical:
     *  if the other party supports SCAN_QR, we should show a QR code in the UI, and vice versa.
     *  For methods that need to be supported by both ends, use the `methods` property.
     *  @param {string} method the method to check
+     *  @param {boolean} force to check even if the phase is not ready or started yet, internal usage
     *  @return {bool} whether or not the other party said the supported the method */
-    otherPartySupportsMethod(method) {
-        if (!this.ready && !this.started) {
+    otherPartySupportsMethod(method, force = false) {
+        if (!force && !this.ready && !this.started) {
            return false;
        }
        const theirMethodEvent = this._eventsByThem.get(REQUEST_TYPE) ||
@@ -286,6 +322,10 @@ export class VerificationRequest extends EventEmitter {
        return this.channel.userId;
    }

+    get isSelfVerification() {
+        return this._client.getUserId() === this.otherUserId;
+    }
+
    /**
     * The id of the user that cancelled the request,
     * only defined when phase is PHASE_CANCELLED
@@ -315,14 +355,6 @@ export class VerificationRequest extends EventEmitter {
        return this._observeOnly;
    }

-    /**
-     * The unpadded base64 encoded shared secret. Primarily used for QR code
-     * verification.
-     */
-    get encodedSharedSecret() {
-        if (!this._sharedSecret) this._generateSharedSecret();
-        return this._sharedSecret;
-    }

    /**
     * Gets which device the verification should be started with
@@ -369,6 +401,7 @@ export class VerificationRequest extends EventEmitter {
                if (!this._verifier) {
                    throw newUnknownMethodError();
                }
+                this._chosenMethod = method;
            }
        }
        return this._verifier;
@@ -382,7 +415,6 @@ export class VerificationRequest extends EventEmitter {
        if (!this.observeOnly && this._phase === PHASE_UNSENT) {
            const methods = [...this._verificationMethods.keys()];
            await this.channel.send(REQUEST_TYPE, {methods});
-            this._generateSharedSecret();
        }
    }

@@ -415,16 +447,9 @@ export class VerificationRequest extends EventEmitter {
            this._accepting = true;
            this.emit("change");
            await this.channel.send(READY_TYPE, {methods});
-            this._generateSharedSecret();
        }
    }

-    _generateSharedSecret() {
-        const secretBytes = new Uint8Array(8);
-        global.crypto.getRandomValues(secretBytes);
-        this._sharedSecret = olmlib.encodeUnpaddedBase64(secretBytes);
-    }
-
    /**
     * Can be used to listen for state changes until the callback returns true.
     * @param {Function} fn callback to evaluate whether the request is in the desired state.
@@ -520,7 +545,7 @@ export class VerificationRequest extends EventEmitter {
        }

        const cancelEvent = this._getEventByEither(CANCEL_TYPE);
-        if (cancelEvent && phase() !== PHASE_DONE) {
+        if ((this._cancelled || cancelEvent) && phase() !== PHASE_DONE) {
            transitions.push({phase: PHASE_CANCELLED, event: cancelEvent});
            return transitions;
        }
@@ -559,6 +584,14 @@ export class VerificationRequest extends EventEmitter {
            const {method} = event.getContent();
            if (!this._verifier && !this.observeOnly) {
                this._verifier = this._createVerifier(method, event);
+                if (!this._verifier) {
+                    this.cancel({
+                        code: "m.unknown_method",
+                        reason: `Unknown method: ${method}`,
+                    });
+                } else {
+                    this._chosenMethod = method;
+                }
            }
        }
    }
@@ -580,10 +613,9 @@ export class VerificationRequest extends EventEmitter {
            return false;
        }
        const oldEvent = this._verifier.startEvent;
-        const isSelfVerification = this.channel.userId === this._client.getUserId();

        let oldRaceIdentifier;
-        if (isSelfVerification) {
+        if (this.isSelfVerification) {
            // if the verifier does not have a startEvent,
            // it is because it's still sending and we are on the initator side
            // we know we are sending a .start event because we already
@@ -603,7 +635,7 @@ export class VerificationRequest extends EventEmitter {
        }

        let newRaceIdentifier;
-        if (isSelfVerification) {
+        if (this.isSelfVerification) {
            const newContent = newEvent.getContent();
            newRaceIdentifier = newContent && newContent.from_device;
        } else {
@@ -612,6 +644,20 @@ export class VerificationRequest extends EventEmitter {
        return newRaceIdentifier < oldRaceIdentifier;
    }

+    hasEventId(eventId) {
+        for (const event of this._eventsByUs.values()) {
+            if (event.getId() === eventId) {
+                return true;
+            }
+        }
+        for (const event of this._eventsByThem.values()) {
+            if (event.getId() === eventId) {
+                return true;
+            }
+        }
+        return false;
+    }
+
    /**
     * Changes the state of the request and verifier in response to a key verification event.
     * @param {string} type the "symbolic" event type, as returned by the `getEventType` function on the channel.
@@ -637,6 +683,18 @@ export class VerificationRequest extends EventEmitter {
            }
        }

+        // This assumes verification won't need to send an event with
+        // the same type for the same party twice.
+        // This is true for QR and SAS verification, and was
+        // added here to prevent verification getting cancelled
+        // when the server duplicates an event (https://github.com/matrix-org/synapse/issues/3365)
+        const isDuplicateEvent = isSentByUs ?
+            this._eventsByUs.has(type) :
+            this._eventsByThem.has(type);
+        if (isDuplicateEvent) {
+            return;
+        }
+
        const oldPhase = this.phase;
        this._addEvent(type, event, isSentByUs);

@@ -650,7 +708,7 @@ export class VerificationRequest extends EventEmitter {
                if (this._verifier.canSwitchStartEvent(event) && newEventWinsRace) {
                    this._verifier.switchStartEvent(event);
                } else if (!isRemoteEcho) {
-                     if (type === CANCEL_TYPE || (this._verifier.events
+                    if (type === CANCEL_TYPE || (this._verifier.events
                        && this._verifier.events.includes(type))) {
                        this._verifier.handleEvent(event);
                    }
@@ -658,6 +716,20 @@ export class VerificationRequest extends EventEmitter {
            }

            if (newTransitions.length) {
+                // create QRCodeData if the other side can scan
+                // important this happens before emitting a phase change,
+                // so listeners can rely on it being there already
+                // We only do this for live events because it is important that
+                // we sign the keys that were in the QR code, and not the keys
+                // we happen to have at some later point in time.
+                if (isLiveEvent && newTransitions.some(t => t.phase === PHASE_READY)) {
+                    const shouldGenerateQrCode =
+                        this.otherPartySupportsMethod(SCAN_QR_CODE_METHOD, true);
+                    if (shouldGenerateQrCode) {
+                        this._qrCodeData = await QRCodeData.create(this, this._client);
+                    }
+                }
+
                const lastTransition = newTransitions[newTransitions.length - 1];
                const {phase} = lastTransition;

@@ -682,7 +754,7 @@ export class VerificationRequest extends EventEmitter {

    _setupTimeout(phase) {
        const shouldTimeout = !this._timeoutTimer && !this.observeOnly &&
-            phase === PHASE_REQUESTED && this.initiatedByMe;
+            phase === PHASE_REQUESTED;

        if (shouldTimeout) {
            this._timeoutTimer = setTimeout(this._cancelOnTimeout, this.timeout);
@@ -701,7 +773,17 @@ export class VerificationRequest extends EventEmitter {

    _cancelOnTimeout = () => {
        try {
-            this.cancel({reason: "Other party didn't accept in time", code: "m.timeout"});
+            if (this.initiatedByMe) {
+                this.cancel({
+                    reason: "Other party didn't accept in time",
+                    code: "m.timeout",
+                });
+            } else {
+                this.cancel({
+                    reason: "User didn't accept in time",
+                    code: "m.timeout",
+                });
+            }
        } catch (err) {
            logger.error("Error while cancelling verification request", err);
        }
@@ -738,16 +820,8 @@ export class VerificationRequest extends EventEmitter {
        if (!isLiveEvent) {
            this._observeOnly = true;
        }
-        // a timestamp is not provided on all to_device events
-        const timestamp = this.channel.getTimestamp(event);
-        if (Number.isFinite(timestamp)) {
-            const elapsed = Date.now() - timestamp;
-            // don't allow interaction on old requests
-            if (elapsed > (VERIFICATION_REQUEST_TIMEOUT - VERIFICATION_REQUEST_MARGIN) ||
-                elapsed < -(VERIFICATION_REQUEST_TIMEOUT / 2)
-            ) {
-                this._observeOnly = true;
-            }
+        if (this.calculateEventTimeout(event) < VERIFICATION_REQUEST_MARGIN) {
+            this._observeOnly = true;
        }
    }

@@ -766,6 +840,8 @@ export class VerificationRequest extends EventEmitter {
                    this._eventsByThem.delete(type);
                }
            }
+            // also remember when we received the request event
+            this._requestReceivedAt = Date.now();
        }
    }

@@ -806,15 +882,26 @@ export class VerificationRequest extends EventEmitter {
        return true;
    }

-    onVerifierFinished() {
-        if (this.channel.needsDoneMessage) {
-            // verification in DM requires a done message
-            this.channel.send("m.key.verification.done", {});
-        }
-        this._verifierHasFinished = true;
+    onVerifierCancelled() {
+        this._cancelled = true;
+        // move to cancelled phase
        const newTransitions = this._applyPhaseTransitions();
        if (newTransitions.length) {
            this._setPhase(newTransitions[newTransitions.length - 1].phase);
        }
    }
+
+    onVerifierFinished() {
+        this.channel.send("m.key.verification.done", {});
+        this._verifierHasFinished = true;
+        // move to .done phase
+        const newTransitions = this._applyPhaseTransitions();
+        if (newTransitions.length) {
+            this._setPhase(newTransitions[newTransitions.length - 1].phase);
+        }
+    }
+
+    getEventFromOtherParty(type) {
+        return this._eventsByThem.get(type);
+    }
 }
@@ -108,8 +108,9 @@ FilterComponent.prototype._checkFields =
        }

        const allowed_values = self[name];
-        if (allowed_values) {
-            if (!allowed_values.map(match_func)) {
+        if (allowed_values && allowed_values.length > 0) {
+            const anyMatch = allowed_values.some(match_func);
+            if (!anyMatch) {
                return false;
            }
        }
@@ -56,13 +56,6 @@ Filter.LAZY_LOADING_MESSAGES_FILTER = {
    lazy_load_members: true,
 };

-Filter.LAZY_LOADING_SYNC_FILTER = {
-    room: {
-        state: Filter.LAZY_LOADING_MESSAGES_FILTER,
-    },
-};
-
-
 /**
 * Get the ID of this filter on your homeserver (if known)
 * @return {?Number} The filter ID
@@ -96,6 +89,7 @@ Filter.prototype.setDefinition = function(definition) {
    //     "state": {
    //       "types": ["m.room.*"],
    //       "not_rooms": ["!726s6s6q:example.com"],
+    //       "lazy_load_members": true,
    //     },
    //     "timeline": {
    //       "limit": 10,
@@ -177,6 +171,10 @@ Filter.prototype.setTimelineLimit = function(limit) {
    setProp(this.definition, "room.timeline.limit", limit);
 };

+Filter.prototype.setLazyLoadMembers = function(enabled) {
+    setProp(this.definition, "room.state.lazy_load_members", !!enabled);
+};
+
 /**
 * Control whether left rooms should be included in responses.
 * @param {boolean} includeLeave True to make rooms the user has left appear
@@ -276,6 +276,9 @@ MatrixHttpApi.prototype = {
                        callbacks.clearTimeout(xhr.timeout_timer);
                        var resp;
                        try {
+                            if (xhr.status === 0) {
+                                throw new AbortError();
+                            }
                            if (!xhr.responseText) {
                                throw new Error('No response body.');
                            }
@@ -789,6 +792,17 @@ const requestCallback = function(
    userDefinedCallback = userDefinedCallback || function() {};

    return function(err, response, body) {
+        if (err) {
+            // the unit tests use matrix-mock-request, which throw the string "aborted" when aborting a request.
+            // See https://github.com/matrix-org/matrix-mock-request/blob/3276d0263a561b5b8326b47bae720578a2c7473a/src/index.js#L48
+            const aborted = err.name === "AbortError" || err === "aborted";
+            if (!aborted && !(err instanceof MatrixError)) {
+                // browser-request just throws normal Error objects,
+                // not `TypeError`s like fetch does. So just assume any
+                // error is due to the connection.
+                err = new ConnectionError("request failed", err);
+            }
+        }
        if (!err) {
            try {
                if (response.statusCode >= 400) {
@@ -876,7 +890,7 @@ function getResponseContentType(response) {

    try {
        return parseContentType(contentType);
-    } catch(e) {
+    } catch (e) {
        throw new Error(`Error parsing Content-Type '${contentType}': ${e}`);
    }
 }
@@ -892,12 +906,76 @@ function getResponseContentType(response) {
 * @prop {Object} data The raw Matrix error JSON used to construct this object.
 * @prop {integer} httpStatus The numeric HTTP status code given
 */
-export function MatrixError(errorJson) {
-    errorJson = errorJson || {};
-    this.errcode = errorJson.errcode;
-    this.name = errorJson.errcode || "Unknown error code";
-    this.message = errorJson.error || "Unknown message";
-    this.data = errorJson;
+export class MatrixError extends Error {
+    constructor(errorJson) {
+        errorJson = errorJson || {};
+        super(`MatrixError: ${errorJson.errcode}`);
+        this.errcode = errorJson.errcode;
+        this.name = errorJson.errcode || "Unknown error code";
+        this.message = errorJson.error || "Unknown message";
+        this.data = errorJson;
+    }
+}
+
+/**
+ * Construct a ConnectionError. This is a JavaScript Error indicating
+ * that a request failed because of some error with the connection, either
+ * CORS was not correctly configured on the server, the server didn't response,
+ * the request timed out, or the internet connection on the client side went down.
+ * @constructor
+ */
+export class ConnectionError extends Error {
+    constructor(message, cause = undefined) {
+        super(message + (cause ? `: ${cause.message}` : ""));
+        this._cause = cause;
+    }
+
+    get name() {
+        return "ConnectionError";
+    }
+
+    get cause() {
+        return this._cause;
+    }
+}
+
+export class AbortError extends Error {
+    constructor() {
+        super("Operation aborted");
+    }
+
+    get name() {
+        return "AbortError";
+    }
+}
+
+/**
+ * Retries a network operation run in a callback.
+ * @param  {number}   maxAttempts maximum attempts to try
+ * @param  {Function} callback    callback that returns a promise of the network operation. If rejected with ConnectionError, it will be retried by calling the callback again.
+ * @return {any} the result of the network operation
+ * @throws {ConnectionError} If after maxAttempts the callback still throws ConnectionError
+ */
+export async function retryNetworkOperation(maxAttempts, callback) {
+    let attempts = 0;
+    let lastConnectionError = null;
+    while (attempts < maxAttempts) {
+        try {
+            if (attempts > 0) {
+                const timeout = 1000 * Math.pow(2, attempts);
+                console.log(`network operation failed ${attempts} times,` +
+                    ` retrying in ${timeout}ms...`);
+                await new Promise(r => setTimeout(r, timeout));
+            }
+            return await callback();
+        } catch (err) {
+            if (err instanceof ConnectionError) {
+                attempts += 1;
+                lastConnectionError = err;
+            } else {
+                throw err;
+            }
+        }
+    }
+    throw lastConnectionError;
 }
-MatrixError.prototype = Object.create(Error.prototype);
-MatrixError.prototype.constructor = MatrixError;
@@ -22,6 +22,7 @@ matrixcs.request(request);
 utils.runPolyfills();

 try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
    const crypto = require('crypto');
    utils.setCrypto(crypto);
 } catch (err) {
@@ -143,11 +143,19 @@ InteractiveAuth.prototype = {
            this._resolveFunc = resolve;
            this._rejectFunc = reject;

-            // if we have no flows, try a request (we'll have
-            // just a session ID in _data if resuming)
-            if (!this._data.flows) {
+            const hasFlows = this._data && this._data.flows;
+
+            // if we have no flows, try a request to acquire the flows
+            if (!hasFlows) {
                if (this._busyChangedCallback) this._busyChangedCallback(true);
-                this._doRequest(this._data).finally(() => {
+                // use the existing sessionid, if one is present.
+                let auth = null;
+                if (this._data.session) {
+                    auth = {
+                        session: this._data.session,
+                    };
+                }
+                this._doRequest(auth).finally(() => {
                    if (this._busyChangedCallback) this._busyChangedCallback(false);
                });
            } else {
@@ -163,6 +171,8 @@ InteractiveAuth.prototype = {
     */
    poll: async function() {
        if (!this._data.session) return;
+        // likewise don't poll if there is no auth session in progress
+        if (!this._resolveFunc) return;
        // if we currently have a request in flight, there's no point making
        // another just to check what the status is
        if (this._submitPromise) return;
@@ -184,7 +194,11 @@ InteractiveAuth.prototype = {
                }
                authDict = {
                    type: EMAIL_STAGE_TYPE,
+                    // TODO: Remove `threepid_creds` once servers support proper UIA
+                    // See https://github.com/matrix-org/synapse/issues/5665
+                    // See https://github.com/matrix-org/matrix-doc/issues/2220
                    threepid_creds: creds,
+                    threepidCreds: creds,
                };
            }
        }
@@ -262,11 +276,16 @@ InteractiveAuth.prototype = {
            }
        }

-        // use the sessionid from the last request.
-        const auth = {
-            session: this._data.session,
-        };
-        utils.extend(auth, authData);
+        // use the sessionid from the last request, if one is present.
+        let auth;
+        if (this._data.session) {
+            auth = {
+                session: this._data.session,
+            };
+            utils.extend(auth, authData);
+        } else {
+            auth = authData;
+        }

        try {
            // NB. the 'background' flag is deprecated by the busyChanged
@@ -318,10 +337,12 @@ InteractiveAuth.prototype = {
        try {
            const result = await this._requestCallback(auth, background);
            this._resolveFunc(result);
+            this._resolveFunc = null;
+            this._rejectFunc = null;
        } catch (error) {
            // sometimes UI auth errors don't come with flows
            const errorFlows = error.data ? error.data.flows : null;
-            const haveFlows = Boolean(this._data.flows) || Boolean(errorFlows);
+            const haveFlows = this._data.flows || Boolean(errorFlows);
            if (error.httpStatus !== 401 || !error.data || !haveFlows) {
                // doesn't look like an interactive-auth failure.
                if (!background) {
@@ -347,7 +368,13 @@ InteractiveAuth.prototype = {
                error.data.session = this._data.session;
            }
            this._data = error.data;
-            this._startNextAuthStage();
+            try {
+                this._startNextAuthStage();
+            } catch (e) {
+                this._rejectFunc(e);
+                this._resolveFunc = null;
+                this._rejectFunc = null;
+            }

            if (
                !this._emailSid &&
@@ -379,8 +406,10 @@ InteractiveAuth.prototype = {
                    // (or not being registered, depending on what we're trying
                    // to do) or it could be a network failure. Either way, pass
                    // the failure up as the user can't complete auth if we can't
-                    // send the email, foe whatever reason.
+                    // send the email, for whatever reason.
                    this._rejectFunc(e);
+                    this._resolveFunc = null;
+                    this._rejectFunc = null;
                } finally {
                    this._requestingEmailToken = false;
                }
@@ -408,7 +437,7 @@ InteractiveAuth.prototype = {
            return;
        }

-        if (this._data.errcode || this._data.error) {
+        if (this._data && this._data.errcode || this._data.error) {
            this._stateUpdatedCallback(nextStage, {
                errcode: this._data.errcode || "",
                error: this._data.error || "",
@@ -16,8 +16,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

+import type Request from "request";
+
 import {MemoryCryptoStore} from "./crypto/store/memory-crypto-store";
+import {LocalStorageCryptoStore} from "./crypto/store/localStorage-crypto-store";
+import {IndexedDBCryptoStore} from "./crypto/store/indexeddb-crypto-store";
 import {MemoryStore} from "./store/memory";
+import {StubStore} from "./store/stub";
+import {LocalIndexedDBStoreBackend} from "./store/indexeddb-local-backend";
+import {RemoteIndexedDBStoreBackend} from "./store/indexeddb-remote-backend";
 import {MatrixScheduler} from "./scheduler";
 import {MatrixClient} from "./client";

@@ -89,6 +96,10 @@ export function wrapRequest(wrapper) {
    };
 }

+type Store =
+    StubStore | MemoryStore | LocalIndexedDBStoreBackend | RemoteIndexedDBStoreBackend;
+
+type CryptoStore = MemoryCryptoStore | LocalStorageCryptoStore | IndexedDBCryptoStore;

 let cryptoStoreFactory = () => new MemoryCryptoStore;

@@ -102,6 +113,21 @@ export function setCryptoStoreFactory(fac) {
    cryptoStoreFactory = fac;
 }

+interface ICreateClientOpts {
+    baseUrl: string;
+    idBaseUrl?: string;
+    store?: Store;
+    cryptoStore?: CryptoStore;
+    scheduler?: MatrixScheduler;
+    request?: Request;
+    userId?: string;
+    accessToken?: string;
+    identityServer?: any;
+    localTimeoutMs?: number;
+    useAuthorizationHeader?: boolean;
+    queryParams?: Record<string, unknown>;
+}
+
 /**
 * Construct a Matrix Client. Similar to {@link module:client.MatrixClient}
 * except that the 'request', 'store' and 'scheduler' dependencies are satisfied.
@@ -125,10 +151,10 @@ export function setCryptoStoreFactory(fac) {
 * @see {@link module:client.MatrixClient} for the full list of options for
 * <code>opts</code>.
 */
-export function createClient(opts) {
+export function createClient(opts: ICreateClientOpts | string) {
    if (typeof opts === "string") {
        opts = {
-            "baseUrl": opts,
+            "baseUrl": opts as string,
        };
    }
    opts.request = opts.request || requestInstance;
@@ -167,7 +193,7 @@ export function createClient(opts) {
 * @param {requestCallback} callback The request callback.
 */

- /**
+/**
  * The request callback interface for performing HTTP requests. This matches the
  * API for the {@link https://github.com/request/request#requestoptions-callback|
  * request NPM module}. The SDK will implement a callback which meets this
@@ -647,7 +647,8 @@ EventTimelineSet.prototype.compareEventOrdering = function(eventId1, eventId2) {
    if (timeline1 === timeline2) {
        // both events are in the same timeline - figure out their
        // relative indices
-        let idx1, idx2;
+        let idx1;
+        let idx2;
        const events = timeline1.getEvents();
        for (let idx = 0; idx < events.length &&
             (idx1 === undefined || idx2 === undefined); idx++) {
@@ -87,7 +87,7 @@ export const MatrixEvent = function(
    // amount of needless string duplication. This can save moderate amounts of
    // memory (~10% on a 350MB heap).
    // 'membership' at the event level (rather than the content level) is a legacy
-    // field that Riot never otherwise looks at, but it will still take up a lot
+    // field that Element never otherwise looks at, but it will still take up a lot
    // of space if we don't intern it.
    ["state_key", "type", "sender", "room_id", "membership"].forEach((prop) => {
        if (!event[prop]) {
@@ -144,6 +144,10 @@ export const MatrixEvent = function(
     */
    this._forwardingCurve25519KeyChain = [];

+    /* where the decryption key is untrusted
+     */
+    this._untrusted = null;
+
    /* if we have a process decrypting this event, a Promise which resolves
     * when it is finished. Normally null.
     */
@@ -160,12 +164,16 @@ export const MatrixEvent = function(
     * so it can be easily accessed from the timeline.
     */
    this.verificationRequest = null;
+
+    /* The txnId with which this event was sent if it was during this session,
+       allows for a unique ID which does not change when the event comes back down sync.
+     */
+    this._txnId = null;
 };
 utils.inherits(MatrixEvent, EventEmitter);


 utils.extend(MatrixEvent.prototype, {
-
    /**
     * Get the event_id for this event.
     * @return {string} The event ID, e.g. <code>$143350589368169JsLZx:localhost
@@ -390,11 +398,12 @@ utils.extend(MatrixEvent.prototype, {
     * @internal
     *
     * @param {module:crypto} crypto crypto module
+     * @param {bool} isRetry True if this is a retry (enables more logging)
     *
     * @returns {Promise} promise which resolves (to undefined) when the decryption
     * attempt is completed.
     */
-    attemptDecryption: async function(crypto) {
+    attemptDecryption: async function(crypto, isRetry) {
        // start with a couple of sanity checks.
        if (!this.isEncrypted()) {
            throw new Error("Attempt to decrypt event which isn't encrypted");
@@ -406,7 +415,7 @@ utils.extend(MatrixEvent.prototype, {
        ) {
            // we may want to just ignore this? let's start with rejecting it.
            throw new Error(
-                "Attempt to decrypt event which has already been encrypted",
+                "Attempt to decrypt event which has already been decrypted",
            );
        }

@@ -424,7 +433,7 @@ utils.extend(MatrixEvent.prototype, {
            return this._decryptionPromise;
        }

-        this._decryptionPromise = this._decryptionLoop(crypto);
+        this._decryptionPromise = this._decryptionLoop(crypto, isRetry);
        return this._decryptionPromise;
    },

@@ -469,7 +478,7 @@ utils.extend(MatrixEvent.prototype, {
        return recipients;
    },

-    _decryptionLoop: async function(crypto) {
+    _decryptionLoop: async function(crypto, isRetry) {
        // make sure that this method never runs completely synchronously.
        // (doing so would mean that we would clear _decryptionPromise *before*
        // it is set in attemptDecryption - and hence end up with a stuck
@@ -486,13 +495,18 @@ utils.extend(MatrixEvent.prototype, {
                    res = this._badEncryptedMessage("Encryption not enabled");
                } else {
                    res = await crypto.decryptEvent(this);
+                    if (isRetry) {
+                        logger.info(`Decrypted event on retry (id=${this.getId()})`);
+                    }
                }
            } catch (e) {
                if (e.name !== "DecryptionError") {
                    // not a decryption error: log the whole exception as an error
                    // (and don't bother with a retry)
+                    const re = isRetry ? 're' : '';
                    logger.error(
-                        `Error decrypting event (id=${this.getId()}): ${e.stack || e}`,
+                        `Error ${re}decrypting event ` +
+                        `(id=${this.getId()}): ${e.stack || e}`,
                    );
                    this._decryptionPromise = null;
                    this._retryDecryption = false;
@@ -593,6 +607,7 @@ utils.extend(MatrixEvent.prototype, {
            decryptionResult.claimedEd25519Key || null;
        this._forwardingCurve25519KeyChain =
            decryptionResult.forwardingCurve25519KeyChain || [];
+        this._untrusted = decryptionResult.untrusted || false;
    },

    /**
@@ -611,7 +626,7 @@ utils.extend(MatrixEvent.prototype, {
     * @return {boolean} True if this event is encrypted.
     */
    isEncrypted: function() {
-        return this.event.type === "m.room.encrypted";
+        return !this.isState() && this.event.type === "m.room.encrypted";
    },

    /**
@@ -683,6 +698,16 @@ utils.extend(MatrixEvent.prototype, {
        return this._forwardingCurve25519KeyChain;
    },

+    /**
+     * Whether the decryption key was obtained from an untrusted source. If so,
+     * we cannot verify the authenticity of the message.
+     *
+     * @return {boolean}
+     */
+    isKeySourceUntrusted: function() {
+        return this._untrusted;
+    },
+
    getUnsigned: function() {
        return this.event.unsigned || {};
    },
@@ -1064,6 +1089,14 @@ utils.extend(MatrixEvent.prototype, {
    setVerificationRequest: function(request) {
        this.verificationRequest = request;
    },
+
+    setTxnId(txnId) {
+        this._txnId = txnId;
+    },
+
+    getTxnId() {
+        return this._txnId;
+    },
 });


@@ -20,7 +20,7 @@ limitations under the License.
 */

 import {EventEmitter} from "events";
-import {getHttpUriForMxc, getIdenticonUri} from "../content-repo";
+import {getHttpUriForMxc} from "../content-repo";
 import * as utils from "../utils";

 /**
@@ -274,10 +274,6 @@ RoomMember.prototype.getAvatarUrl =
    );
    if (httpUrl) {
        return httpUrl;
-    } else if (allowDefault) {
-        return getIdenticonUri(
-            baseUrl, this.userId, width, height,
-        );
    }
    return null;
 };
@@ -286,9 +282,9 @@ RoomMember.prototype.getAvatarUrl =
 * @return {string} the mxc avatar url
 */
 RoomMember.prototype.getMxcAvatarUrl = function() {
-    if(this.events.member) {
+    if (this.events.member) {
        return this.events.member.getDirectionalContent().avatar_url;
-    } else if(this.user) {
+    } else if (this.user) {
        return this.user.avatarUrl;
    }
    return null;
@@ -68,9 +68,7 @@ export function RoomState(roomId, oobMemberFlags = undefined) {
    this.members = {
        // userId: RoomMember
    };
-    this.events = {
-        // eventType: { stateKey: MatrixEvent }
-    };
+    this.events = new Map(); // Map<eventType, Map<stateKey, MatrixEvent>>
    this.paginationToken = null;

    this._sentinels = {
@@ -211,14 +209,14 @@ RoomState.prototype.getSentinelMember = function(userId) {
 * <code>undefined</code>, else a single event (or null if no match found).
 */
 RoomState.prototype.getStateEvents = function(eventType, stateKey) {
-    if (!this.events[eventType]) {
+    if (!this.events.has(eventType)) {
        // no match
        return stateKey === undefined ? [] : null;
    }
    if (stateKey === undefined) { // return all values
-        return utils.values(this.events[eventType]);
+        return Array.from(this.events.get(eventType).values());
    }
-    const event = this.events[eventType][stateKey];
+    const event = this.events.get(eventType).get(stateKey);
    return event ? event : null;
 };

@@ -238,9 +236,8 @@ RoomState.prototype.clone = function() {
    const status = this._oobMemberFlags.status;
    this._oobMemberFlags.status = OOB_STATUS_NOTSTARTED;

-    Object.values(this.events).forEach((eventsByStateKey) => {
-        const eventsForType = Object.values(eventsByStateKey);
-        copy.setStateEvents(eventsForType);
+    Array.from(this.events.values()).forEach((eventsByStateKey) => {
+        copy.setStateEvents(Array.from(eventsByStateKey.values()));
    });

    // Ugly hack: see above
@@ -276,8 +273,8 @@ RoomState.prototype.clone = function() {
 */
 RoomState.prototype.setUnknownStateEvents = function(events) {
    const unknownStateEvents = events.filter((event) => {
-        return this.events[event.getType()] === undefined ||
-            this.events[event.getType()][event.getStateKey()] === undefined;
+        return !this.events.has(event.getType()) ||
+            !this.events.get(event.getType()).has(event.getStateKey());
    });

    this.setStateEvents(unknownStateEvents);
@@ -306,6 +303,7 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
            return;
        }

+        const lastStateEvent = self._getStateEventMatching(event);
        self._setStateEvent(event);
        if (event.getType() === "m.room.member") {
            _updateDisplayNameCache(
@@ -313,7 +311,7 @@ RoomState.prototype.setStateEvents = function(stateEvents) {
            );
            _updateThirdPartyTokenCache(self, event);
        }
-        self.emit("RoomState.events", event, self);
+        self.emit("RoomState.events", event, self, lastStateEvent);
    });

    // update higher level data structures. This needs to be done AFTER the
@@ -385,10 +383,15 @@ RoomState.prototype._getOrCreateMember = function(userId, event) {
 };

 RoomState.prototype._setStateEvent = function(event) {
-    if (this.events[event.getType()] === undefined) {
-        this.events[event.getType()] = {};
+    if (!this.events.has(event.getType())) {
+        this.events.set(event.getType(), new Map());
    }
-    this.events[event.getType()][event.getStateKey()] = event;
+    this.events.get(event.getType()).set(event.getStateKey(), event);
+};
+
+RoomState.prototype._getStateEventMatching = function(event) {
+    if (!this.events.has(event.getType())) return null;
+    return this.events.get(event.getType()).get(event.getStateKey());
 };

 RoomState.prototype._updateMember = function(member) {
@@ -670,7 +673,7 @@ RoomState.prototype._maySendEventOfType = function(eventType, userId, state) {
        const userPowerLevel = power_levels.users && power_levels.users[userId];
        if (Number.isFinite(userPowerLevel)) {
            powerLevel = userPowerLevel;
-        } else if(Number.isFinite(power_levels.users_default)) {
+        } else if (Number.isFinite(power_levels.users_default)) {
            powerLevel = power_levels.users_default;
        }

@@ -769,8 +772,12 @@ function _updateDisplayNameCache(roomState, userId, displayName) {
 * @param {MatrixEvent} event The matrix event which caused this event to fire.
 * @param {RoomState} state The room state whose RoomState.events dictionary
 * was updated.
+ * @param {MatrixEvent} prevEvent The event being replaced by the new state, if
+ * known. Note that this can differ from `getPrevContent()` on the new state event
+ * as this is the store's view of the last state, not the previous state provided
+ * by the server.
 * @example
- * matrixClient.on("RoomState.events", function(event, state){
+ * matrixClient.on("RoomState.events", function(event, state, prevEvent){
 *   var newStateEvent = event;
 * });
 */
@@ -23,7 +23,7 @@ limitations under the License.
 import {EventEmitter} from "events";
 import {EventTimelineSet} from "./event-timeline-set";
 import {EventTimeline} from "./event-timeline";
-import {getHttpUriForMxc, getIdenticonUri} from "../content-repo";
+import {getHttpUriForMxc} from "../content-repo";
 import * as utils from "../utils";
 import {EventStatus, MatrixEvent} from "./event";
 import {RoomMember} from "./room-member";
@@ -122,6 +122,10 @@ export function Room(roomId, client, myUserId, opts) {
    opts = opts || {};
    opts.pendingEventOrdering = opts.pendingEventOrdering || "chronological";

+    // In some cases, we add listeners for every displayed Matrix event, so it's
+    // common to have quite a few more than the default limit.
+    this.setMaxListeners(100);
+
    this.reEmitter = new ReEmitter(this);

    if (["chronological", "detached"].indexOf(opts.pendingEventOrdering) === -1) {
@@ -209,7 +213,7 @@ utils.inherits(Room, EventEmitter);
 Room.prototype.getVersion = function() {
    const createEvent = this.currentState.getStateEvents("m.room.create", "");
    if (!createEvent) {
-        logger.warn("Room " + this.room_id + " does not have an m.room.create event");
+        logger.warn("Room " + this.roomId + " does not have an m.room.create event");
        return '1';
    }
    const ver = createEvent.getContent()['room_version'];
@@ -675,7 +679,7 @@ Room.prototype.hasUnverifiedDevices = async function() {
    }
    const e2eMembers = await this.getEncryptionTargetMembers();
    for (const member of e2eMembers) {
-        const devices = await this._client.getStoredDevicesForUser(member.userId);
+        const devices = this._client.getStoredDevicesForUser(member.userId);
        if (devices.some((device) => device.isUnverified())) {
            return true;
        }
@@ -814,10 +818,6 @@ Room.prototype.getAvatarUrl = function(baseUrl, width, height, resizeMethod,
        return getHttpUriForMxc(
            baseUrl, mainUrl, width, height, resizeMethod,
        );
-    } else if (allowDefault) {
-        return getIdenticonUri(
-            baseUrl, this.roomId, width, height,
-        );
    }

    return null;
@@ -1278,6 +1278,11 @@ Room.prototype._handleRemoteEcho = function(remoteEvent, localEvent) {
    const newEventId = remoteEvent.getId();
    const oldStatus = localEvent.status;

+    logger.debug(
+        `Got remote echo for event ${oldEventId} -> ${newEventId} ` +
+        `old status ${oldStatus}`,
+    );
+
    // no longer pending
    delete this._txnToEvent[remoteEvent.getUnsigned().transaction_id];

@@ -1347,7 +1352,10 @@ ALLOWED_TRANSITIONS[EventStatus.CANCELLED] =
 * @fires module:client~MatrixClient#event:"Room.localEchoUpdated"
 */
 Room.prototype.updatePendingEvent = function(event, newStatus, newEventId) {
-    logger.log(`setting pendingEvent status to ${newStatus} in ${event.getRoomId()}`);
+    logger.log(
+        `setting pendingEvent status to ${newStatus} in ${event.getRoomId()} ` +
+        `event ID ${event.getId()} -> ${newEventId}`,
+    );

    // if the message was sent, we expect an event id
    if (newStatus == EventStatus.SENT && !newEventId) {
@@ -1785,8 +1793,9 @@ Room.prototype.addAccountData = function(events) {
        if (event.getType() === "m.tag") {
            this.addTags(event);
        }
+        const lastEvent = this.accountData[event.getType()];
        this.accountData[event.getType()] = event;
-        this.emit("Room.accountData", event, this);
+        this.emit("Room.accountData", event, this, lastEvent);
    }
 };

@@ -1891,14 +1900,14 @@ function calculateRoomName(room, userId, ignoreRoomNameEvent) {
    // let's try to figure out who was here before
    let leftNames = otherNames;
    // if we didn't have heroes, try finding them in the room state
-    if(!leftNames.length) {
+    if (!leftNames.length) {
        leftNames = room.currentState.getMembers().filter((m) => {
            return m.userId !== userId &&
                m.membership !== "invite" &&
                m.membership !== "join";
        }).map((m) => m.name);
    }
-    if(leftNames.length) {
+    if (leftNames.length) {
        return `Empty room (was ${memberNamesToRoomName(leftNames)})`;
    } else {
        return "Empty room";
@@ -1983,8 +1992,10 @@ function memberNamesToRoomName(names, count = (names.length + 1)) {
 * @event module:client~MatrixClient#"Room.accountData"
 * @param {event} event The account_data event
 * @param {Room} room The room whose account_data was updated.
+ * @param {MatrixEvent} prevEvent The event being replaced by
+ * the new account data, if known.
 * @example
- * matrixClient.on("Room.accountData", function(event, room){
+ * matrixClient.on("Room.accountData", function(event, room, oldEvent){
 *   if (event.getType() === "m.room.colorscheme") {
 *       applyColorScheme(event.getContents());
 *   }
@@ -129,7 +129,11 @@ User.prototype.setPresenceEvent = function(event) {
 */
 User.prototype.setDisplayName = function(name) {
    const oldName = this.displayName;
-    this.displayName = name;
+    if (typeof name === "string") {
+        this.displayName = name;
+    } else {
+        this.displayName = undefined;
+    }
    if (name !== oldName) {
        this._updateModifiedTime();
    }
@@ -142,7 +146,11 @@ User.prototype.setDisplayName = function(name) {
 * @param {string} name The new display name.
 */
 User.prototype.setRawDisplayName = function(name) {
-    this.rawDisplayName = name;
+    if (typeof name === "string") {
+        this.rawDisplayName = name;
+    } else {
+        this.rawDisplayName = undefined;
+    }
 };


@@ -23,9 +23,8 @@ import {escapeRegExp, globToRegexp, isNullOrUndefined} from "./utils";

 const RULEKINDS_IN_ORDER = ['override', 'content', 'room', 'sender', 'underride'];

-// The default override rules to apply when calculating actions for an event. These
-// defaults apply under no other circumstances to avoid confusing the client with server
-// state. We do this for two reasons:
+// The default override rules to apply to the push rules that arrive from the server.
+// We do this for two reasons:
 //   1. Synapse is unlikely to send us the push rule in an incremental sync - see
 //      https://github.com/matrix-org/synapse/pull/4867#issuecomment-481446072 for
 //      more details.
@@ -85,12 +84,15 @@ export function PushProcessor(client) {
        // $glob: RegExp,
    };

-    const matchingRuleFromKindSet = (ev, kindset, device) => {
+    const matchingRuleFromKindSet = (ev, kindset) => {
        for (let ruleKindIndex = 0;
                ruleKindIndex < RULEKINDS_IN_ORDER.length;
                ++ruleKindIndex) {
            const kind = RULEKINDS_IN_ORDER[ruleKindIndex];
            const ruleset = kindset[kind];
+            if (!ruleset) {
+                continue;
+            }

            for (let ruleIndex = 0; ruleIndex < ruleset.length; ++ruleIndex) {
                const rule = ruleset[ruleIndex];
@@ -98,7 +100,7 @@ export function PushProcessor(client) {
                    continue;
                }

-                const rawrule = templateRuleToRaw(kind, rule, device);
+                const rawrule = templateRuleToRaw(kind, rule);
                if (!rawrule) {
                    continue;
                }
@@ -112,7 +114,7 @@ export function PushProcessor(client) {
        return null;
    };

-    const templateRuleToRaw = function(kind, tprule, device) {
+    const templateRuleToRaw = function(kind, tprule) {
        const rawrule = {
            'rule_id': tprule.rule_id,
            'actions': tprule.actions,
@@ -154,19 +156,12 @@ export function PushProcessor(client) {
                });
                break;
        }
-        if (device) {
-            rawrule.conditions.push({
-                'kind': 'device',
-                'profile_tag': device,
-            });
-        }
        return rawrule;
    };

    const eventFulfillsCondition = function(cond, ev) {
        const condition_functions = {
            "event_match": eventFulfillsEventMatchCondition,
-            "device": eventFulfillsDeviceCondition,
            "contains_display_name": eventFulfillsDisplayNameCondition,
            "room_member_count": eventFulfillsRoomMemberCountCondition,
            "sender_notification_permission": eventFulfillsSenderNotifPermCondition,
@@ -258,10 +253,6 @@ export function PushProcessor(client) {
        return content.body.search(pat) > -1;
    };

-    const eventFulfillsDeviceCondition = function(cond, ev) {
-        return false; // XXX: Allow a profile tag to be set for the web client instance
-    };
-
    const eventFulfillsEventMatchCondition = function(cond, ev) {
        if (!cond.key) {
            return false;
@@ -326,23 +317,13 @@ export function PushProcessor(client) {
    };

    const matchingRuleForEventWithRulesets = function(ev, rulesets) {
-        if (!rulesets || !rulesets.device) {
+        if (!rulesets) {
            return null;
        }
-        if (ev.getSender() == client.credentials.userId) {
+        if (ev.getSender() === client.credentials.userId) {
            return null;
        }

-        const allDevNames = Object.keys(rulesets.device);
-        for (let i = 0; i < allDevNames.length; ++i) {
-            const devname = allDevNames[i];
-            const devrules = rulesets.device[devname];
-
-            const matchingRule = matchingRuleFromKindSet(devrules, devname);
-            if (matchingRule) {
-                return matchingRule;
-            }
-        }
        return matchingRuleFromKindSet(ev, rulesets.global);
    };

@@ -364,33 +345,6 @@ export function PushProcessor(client) {
        return actionObj;
    };

-    const applyRuleDefaults = function(clientRuleset) {
-        // Deep clone the object before we mutate it
-        const ruleset = JSON.parse(JSON.stringify(clientRuleset));
-
-        if (!clientRuleset['global']) {
-            clientRuleset['global'] = {};
-        }
-        if (!clientRuleset['global']['override']) {
-            clientRuleset['global']['override'] = [];
-        }
-
-        // Apply default overrides
-        const globalOverrides = clientRuleset['global']['override'];
-        for (const override of DEFAULT_OVERRIDE_RULES) {
-            const existingRule = globalOverrides
-                .find((r) => r.rule_id === override.rule_id);
-
-            if (!existingRule) {
-                const ruleId = override.rule_id;
-                console.warn(`Adding default global override for ${ruleId}`);
-                globalOverrides.push(override);
-            }
-        }
-
-        return ruleset;
-    };
-
    this.ruleMatchesEvent = function(rule, ev) {
        let ret = true;
        for (let i = 0; i < rule.conditions.length; ++i) {
@@ -410,8 +364,7 @@ export function PushProcessor(client) {
     * @return {PushAction}
     */
    this.actionsForEvent = function(ev) {
-        const rules = applyRuleDefaults(client.pushRules);
-        return pushActionsForEventAndRulesets(ev, rules);
+        return pushActionsForEventAndRulesets(ev, client.pushRules);
    };

    /**
@@ -421,7 +374,7 @@ export function PushProcessor(client) {
     * @return {object} The push rule, or null if no such rule was found
     */
    this.getPushRuleById = function(ruleId) {
-        for (const scope of ['device', 'global']) {
+        for (const scope of ['global']) {
            if (client.pushRules[scope] === undefined) continue;

            for (const kind of RULEKINDS_IN_ORDER) {
@@ -476,18 +429,25 @@ PushProcessor.rewriteDefaultRules = function(incomingRules) {
    if (!newRules.global) newRules.global = {};
    if (!newRules.global.override) newRules.global.override = [];

-    // Fix default override rules
-    newRules.global.override = newRules.global.override.map(r => {
-        const defaultRule = DEFAULT_OVERRIDE_RULES.find(d => d.rule_id === r.rule_id);
-        if (!defaultRule) return r;
+    // Merge the client-level defaults with the ones from the server
+    const globalOverrides = newRules.global.override;
+    for (const override of DEFAULT_OVERRIDE_RULES) {
+        const existingRule = globalOverrides
+            .find((r) => r.rule_id === override.rule_id);

-        // Copy over the actions, default, and conditions. Don't touch the user's
-        // preference.
-        r.default = defaultRule.default;
-        r.conditions = defaultRule.conditions;
-        r.actions = defaultRule.actions;
-        return r;
-    });
+        if (existingRule) {
+            // Copy over the actions, default, and conditions. Don't touch the user's
+            // preference.
+            existingRule.default = override.default;
+            existingRule.conditions = override.conditions;
+            existingRule.actions = override.actions;
+        } else {
+            // Add the rule
+            const ruleId = override.rule_id;
+            console.warn(`Adding default global override for ${ruleId}`);
+            globalOverrides.push(override);
+        }
+    }

    return newRules;
 };
@@ -185,8 +185,8 @@ function _runCallbacks() {
 */
 function binarySearch(array, func) {
    // min is inclusive, max exclusive.
-    let min = 0,
-        max = array.length;
+    let min = 0;
+    let max = array.length;

    while (min < max) {
        const mid = (min + max) >> 1;
@@ -25,6 +25,15 @@ limitations under the License.
 import {User} from "../models/user";
 import * as utils from "../utils";

+function isValidFilterId(filterId) {
+    const isValidStr = typeof filterId === "string" &&
+        !!filterId &&
+        filterId !== "undefined" && // exclude these as we've serialized undefined in localStorage before
+        filterId !== "null";
+
+    return isValidStr || typeof filterId === "number";
+}
+
 /**
 * Construct a new in-memory data store for the Matrix Client.
 * @constructor
@@ -273,8 +282,17 @@ MemoryStore.prototype = {
        if (!this.localStorage) {
            return null;
        }
+        const key = "mxjssdk_memory_filter_" + filterName;
+        // XXX Storage.getItem doesn't throw ...
+        // or are we using something different
+        // than window.localStorage in some cases
+        // that does throw?
+        // that would be very naughty
        try {
-            return this.localStorage.getItem("mxjssdk_memory_filter_" + filterName);
+            const value = this.localStorage.getItem(key);
+            if (isValidFilterId(value)) {
+                return value;
+            }
        } catch (e) {}
        return null;
    },
@@ -288,8 +306,13 @@ MemoryStore.prototype = {
        if (!this.localStorage) {
            return;
        }
+        const key = "mxjssdk_memory_filter_" + filterName;
        try {
-            this.localStorage.setItem("mxjssdk_memory_filter_" + filterName, filterId);
+            if (isValidFilterId(filterId)) {
+                this.localStorage.setItem(key, filterId);
+            } else {
+                this.localStorage.removeItem(key);
+            }
        } catch (e) {}
    },

@@ -511,6 +511,12 @@ SyncApi.prototype.sync = function() {
        checkLazyLoadStatus(); // advance to the next stage
    }

+    function buildDefaultFilter() {
+        const filter = new Filter(client.credentials.userId);
+        filter.setTimelineLimit(self.opts.initialSyncLimit);
+        return filter;
+    }
+
    const checkLazyLoadStatus = async () => {
        debuglog("Checking lazy load status...");
        if (this.opts.lazyLoadMembers && client.isGuest()) {
@@ -520,19 +526,11 @@ SyncApi.prototype.sync = function() {
            debuglog("Checking server lazy load support...");
            const supported = await client.doesServerSupportLazyLoading();
            if (supported) {
-                try {
-                    debuglog("Creating and storing lazy load sync filter...");
-                    this.opts.filter = await client.createFilter(
-                        Filter.LAZY_LOADING_SYNC_FILTER,
-                    );
-                    debuglog("Created and stored lazy load sync filter");
-                } catch (err) {
-                    logger.error(
-                        "Creating and storing lazy load sync filter failed",
-                        err,
-                    );
-                    throw err;
+                debuglog("Enabling lazy load on sync filter...");
+                if (!this.opts.filter) {
+                    this.opts.filter = buildDefaultFilter();
                }
+                this.opts.filter.setLazyLoadMembers(true);
            } else {
                debuglog("LL: lazy loading requested but not supported " +
                    "by server, so disabling");
@@ -575,8 +573,7 @@ SyncApi.prototype.sync = function() {
        if (self.opts.filter) {
            filter = self.opts.filter;
        } else {
-            filter = new Filter(client.credentials.userId);
-            filter.setTimelineLimit(self.opts.initialSyncLimit);
+            filter = buildDefaultFilter();
        }

        let filterId;
@@ -704,7 +701,7 @@ SyncApi.prototype._syncFromCache = async function(savedSync) {

    try {
        await this._processSyncResponse(syncEventData, data);
-    } catch(e) {
+    } catch (e) {
        logger.error("Error processing cached sync", e.stack || e);
    }

@@ -777,7 +774,7 @@ SyncApi.prototype._sync = async function(syncOptions) {

    try {
        await this._processSyncResponse(syncEventData, data);
-    } catch(e) {
+    } catch (e) {
        // log the exception with stack if we have it, else fall back
        // to the plain description
        logger.error("Caught /sync error", e.stack || e);
@@ -897,7 +894,7 @@ SyncApi.prototype._onSyncError = function(err, syncOptions) {
    logger.error("/sync error %s", err);
    logger.error(err);

-    if(this._shouldAbortSync(err)) {
+    if (this._shouldAbortSync(err)) {
        return;
    }

@@ -1026,18 +1023,23 @@ SyncApi.prototype._processSyncResponse = async function(
    // handle non-room account_data
    if (data.account_data && utils.isArray(data.account_data.events)) {
        const events = data.account_data.events.map(client.getEventMapper());
+        const prevEventsMap = events.reduce((m, c) => {
+            m[c.getId()] = client.store.getAccountData(c.getType());
+            return m;
+        }, {});
        client.store.storeAccountDataEvents(events);
        events.forEach(
            function(accountDataEvent) {
                // Honour push rules that come down the sync stream but also
                // honour push rules that were previously cached. Base rules
-                // will be updated when we recieve push rules via getPushRules
+                // will be updated when we receive push rules via getPushRules
                // (see SyncApi.prototype.sync) before syncing over the network.
                if (accountDataEvent.getType() === 'm.push_rules') {
                    const rules = accountDataEvent.getContent();
                    client.pushRules = PushProcessor.rewriteDefaultRules(rules);
                }
-                client.emit("accountData", accountDataEvent);
+                const prevEvent = prevEventsMap[accountDataEvent.getId()];
+                client.emit("accountData", accountDataEvent, prevEvent);
                return accountDataEvent;
            },
        );
@@ -396,7 +396,8 @@ TimelineWindow.prototype.getEvents = function() {
        // (Note that both this._start.index and this._end.index are relative
        // to their respective timelines' BaseIndex).
        //
-        let startIndex = 0, endIndex = events.length;
+        let startIndex = 0;
+        let endIndex = events.length;
        if (timeline === this._start.timeline) {
            startIndex = this._start.index + timeline.getBaseIndex();
        }
@@ -49,7 +49,7 @@ export function encodeParams(params: Record<string, string>): string {
 * @return {string} The result of replacing all template variables e.g. '/foo/baz'.
 */
 export function encodeUri(pathTemplate: string,
-                          variables: Record<string, string>): string {
+    variables: Record<string, string>): string {
    for (const key in variables) {
        if (!variables.hasOwnProperty(key)) {
            continue;
@@ -85,7 +85,7 @@ export function map<T, S>(array: T[], fn: (t: T) => S): S[] {
 * @return {Array} A new array with the results of the function.
 */
 export function filter<T>(array: T[],
-                          fn: (t: T, i?: number, a?: T[]) => boolean): T[] {
+    fn: (t: T, i?: number, a?: T[]) => boolean): T[] {
    const results: T[] = [];
    for (let i = 0; i < array.length; i++) {
        if (fn(array[i], i, array)) {
@@ -151,10 +151,10 @@ export function forEach<T>(array: T[], fn: (t: T, i: number) => void) {
 * the given function.
 */
 export function findElement<T>(
-        array: T[],
-        fn: (t: T, i?: number, a?: T[]) => boolean,
-        reverse?: boolean
-    ) {
+    array: T[],
+    fn: (t: T, i?: number, a?: T[]) => boolean,
+    reverse?: boolean,
+) {
    let i;
    if (reverse) {
        for (i = array.length - 1; i >= 0; i--) {
@@ -182,10 +182,10 @@ export function findElement<T>(
 * @return {boolean} True if an element was removed.
 */
 export function removeElement<T>(
-        array: T[],
-        fn: (t: T, i?: number, a?: T[]) => boolean,
-        reverse?: boolean
-    ) {
+    array: T[],
+    fn: (t: T, i?: number, a?: T[]) => boolean,
+    reverse?: boolean,
+) {
    let i;
    let removed;
    if (reverse) {
@@ -265,7 +265,7 @@ export function checkObjectHasNoAdditionalKeys(obj: object, allowedKeys: string[
 * @param {Object} obj The object to deep copy.
 * @return {Object} A copy of the object without any references to the original.
 */
-export function deepCopy(obj: object): object {
+export function deepCopy<T>(obj: T): T {
    return JSON.parse(JSON.stringify(obj));
 }

@@ -293,7 +293,7 @@ export function deepCompare(x: any, y: any): boolean {

    // special-case NaN (since NaN !== NaN)
    if (typeof x === 'number' && isNaN(x) && isNaN(y)) {
-         return true;
+        return true;
    }

    // special-case null (since typeof null == 'object', but null.constructor
@@ -368,10 +368,11 @@ export function deepCompare(x: any, y: any): boolean {
 *
 * @return {Object} target
 */
-export function extend() {
-    const target = arguments[0] || {};
-    for (let i = 1; i < arguments.length; i++) {
-        const source = arguments[i];
+export function extend(...restParams) {
+    const target = restParams[0] || {};
+    for (let i = 1; i < restParams.length; i++) {
+        const source = restParams[i];
+        if (!source) continue;
        for (const propName in source) { // eslint-disable-line guard-for-in
            target[propName] = source[propName];
        }
@@ -388,36 +389,36 @@ export function runPolyfills() {
    // SOURCE:
    // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter
    if (!Array.prototype.filter) {
-      Array.prototype.filter = function(fun: Function/*, thisArg*/) {
-        if (this === void 0 || this === null) {
-          throw new TypeError();
-        }
-
-        const t = Object(this);
-        const len = t.length >>> 0;
-        if (typeof fun !== 'function') {
-          throw new TypeError();
-        }
-
-        const res = [];
-        const thisArg = arguments.length >= 2 ? arguments[1] : void 0;
-        for (let i = 0; i < len; i++) {
-          if (i in t) {
-            const val = t[i];
-
-            // NOTE: Technically this should Object.defineProperty at
-            //       the next index, as push can be affected by
-            //       properties on Object.prototype and Array.prototype.
-            //       But that method's new, and collisions should be
-            //       rare, so use the more-compatible alternative.
-            if (fun.call(thisArg, val, i, t)) {
-              res.push(val);
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.filter = function(fun: Function/*, thisArg*/, ...restProps) {
+            if (this === void 0 || this === null) {
+                throw new TypeError();
            }
-          }
-        }

-        return res;
-      };
+            const t = Object(this);
+            const len = t.length >>> 0;
+            if (typeof fun !== 'function') {
+                throw new TypeError();
+            }
+
+            const res = [];
+            const thisArg = restProps ? restProps[0] : void 0;
+            for (let i = 0; i < len; i++) {
+                if (i in t) {
+                    const val = t[i];
+
+                    // NOTE: Technically this should Object.defineProperty at
+                    //       the next index, as push can be affected by
+                    //       properties on Object.prototype and Array.prototype.
+                    //       But that method's new, and collisions should be
+                    //       rare, so use the more-compatible alternative.
+                    if (fun.call(thisArg, val, i, t)) {
+                        res.push(val);
+                    }
+                }
+            }
+            return res;
+        };
    }

    //                Array.prototype.map
@@ -427,151 +428,156 @@ export function runPolyfills() {
    // Production steps of ECMA-262, Edition 5, 15.4.4.19
    // Reference: http://es5.github.io/#x15.4.4.19
    if (!Array.prototype.map) {
-      Array.prototype.map = function(callback, thisArg) {
-        let T, k;
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.map = function(callback, thisArg) {
+            let T;
+            let k;

-        if (this === null || this === undefined) {
-          throw new TypeError(' this is null or not defined');
-        }
+            if (this === null || this === undefined) {
+                throw new TypeError(' this is null or not defined');
+            }

-        // 1. Let O be the result of calling ToObject passing the |this|
-        //    value as the argument.
-        const O = Object(this);
+            // 1. Let O be the result of calling ToObject passing the |this|
+            //    value as the argument.
+            const O = Object(this);

-        // 2. Let lenValue be the result of calling the Get internal
-        //    method of O with the argument "length".
-        // 3. Let len be ToUint32(lenValue).
-        const len = O.length >>> 0;
+            // 2. Let lenValue be the result of calling the Get internal
+            //    method of O with the argument "length".
+            // 3. Let len be ToUint32(lenValue).
+            const len = O.length >>> 0;

-        // 4. If IsCallable(callback) is false, throw a TypeError exception.
-        // See: http://es5.github.com/#x9.11
-        if (typeof callback !== 'function') {
-          throw new TypeError(callback + ' is not a function');
-        }
+            // 4. If IsCallable(callback) is false, throw a TypeError exception.
+            // See: http://es5.github.com/#x9.11
+            if (typeof callback !== 'function') {
+                throw new TypeError(callback + ' is not a function');
+            }

-        // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
-        if (arguments.length > 1) {
-          T = thisArg;
-        }
+            // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
+            if (arguments.length > 1) {
+                T = thisArg;
+            }

-        // 6. Let A be a new array created as if by the expression new Array(len)
-        //    where Array is the standard built-in constructor with that name and
-        //    len is the value of len.
-        const A = new Array(len);
+            // 6. Let A be a new array created as if by the expression new Array(len)
+            //    where Array is the standard built-in constructor with that name and
+            //    len is the value of len.
+            const A = new Array(len);

-        // 7. Let k be 0
-        k = 0;
+            // 7. Let k be 0
+            k = 0;

-        // 8. Repeat, while k < len
-        while (k < len) {
-          let kValue, mappedValue;
+            // 8. Repeat, while k < len
+            while (k < len) {
+                let kValue;
+                let mappedValue;

-          // a. Let Pk be ToString(k).
-          //   This is implicit for LHS operands of the in operator
-          // b. Let kPresent be the result of calling the HasProperty internal
-          //    method of O with argument Pk.
-          //   This step can be combined with c
-          // c. If kPresent is true, then
-          if (k in O) {
-            // i. Let kValue be the result of calling the Get internal
-            //    method of O with argument Pk.
-            kValue = O[k];
+                // a. Let Pk be ToString(k).
+                //   This is implicit for LHS operands of the in operator
+                // b. Let kPresent be the result of calling the HasProperty internal
+                //    method of O with argument Pk.
+                //   This step can be combined with c
+                // c. If kPresent is true, then
+                if (k in O) {
+                    // i. Let kValue be the result of calling the Get internal
+                    //    method of O with argument Pk.
+                    kValue = O[k];

-            // ii. Let mappedValue be the result of calling the Call internal
-            //     method of callback with T as the this value and argument
-            //     list containing kValue, k, and O.
-            mappedValue = callback.call(T, kValue, k, O);
+                    // ii. Let mappedValue be the result of calling the Call internal
+                    //     method of callback with T as the this value and argument
+                    //     list containing kValue, k, and O.
+                    mappedValue = callback.call(T, kValue, k, O);

-            // iii. Call the DefineOwnProperty internal method of A with arguments
-            // Pk, Property Descriptor
-            // { Value: mappedValue,
-            //   Writable: true,
-            //   Enumerable: true,
-            //   Configurable: true },
-            // and false.
+                    // iii. Call the DefineOwnProperty internal method of A with arguments
+                    // Pk, Property Descriptor
+                    // { Value: mappedValue,
+                    //   Writable: true,
+                    //   Enumerable: true,
+                    //   Configurable: true },
+                    // and false.

-            // In browsers that support Object.defineProperty, use the following:
-            // Object.defineProperty(A, k, {
-            //   value: mappedValue,
-            //   writable: true,
-            //   enumerable: true,
-            //   configurable: true
-            // });
+                    // In browsers that support Object.defineProperty, use the following:
+                    // Object.defineProperty(A, k, {
+                    //   value: mappedValue,
+                    //   writable: true,
+                    //   enumerable: true,
+                    //   configurable: true
+                    // });

-            // For best browser support, use the following:
-            A[k] = mappedValue;
-          }
-          // d. Increase k by 1.
-          k++;
-        }
+                    // For best browser support, use the following:
+                    A[k] = mappedValue;
+                }
+                // d. Increase k by 1.
+                k++;
+            }

-        // 9. return A
-        return A;
-      };
+            // 9. return A
+            return A;
+        };
    }

    //                Array.prototype.forEach
    // ========================================================
    // SOURCE:
-  // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach
+    // https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach
    // Production steps of ECMA-262, Edition 5, 15.4.4.18
    // Reference: http://es5.github.io/#x15.4.4.18
    if (!Array.prototype.forEach) {
-      Array.prototype.forEach = function(callback, thisArg) {
-        let T, k;
+        // eslint-disable-next-line no-extend-native
+        Array.prototype.forEach = function(callback, thisArg) {
+            let T;
+            let k;

-        if (this === null || this === undefined) {
-          throw new TypeError(' this is null or not defined');
-        }
+            if (this === null || this === undefined) {
+                throw new TypeError(' this is null or not defined');
+            }

-        // 1. Let O be the result of calling ToObject passing the |this| value as the
-        // argument.
-        const O = Object(this);
+            // 1. Let O be the result of calling ToObject passing the |this| value as the
+            // argument.
+            const O = Object(this);

-        // 2. Let lenValue be the result of calling the Get internal method of O with the
-        // argument "length".
-        // 3. Let len be ToUint32(lenValue).
-        const len = O.length >>> 0;
+            // 2. Let lenValue be the result of calling the Get internal method of O with the
+            // argument "length".
+            // 3. Let len be ToUint32(lenValue).
+            const len = O.length >>> 0;

-        // 4. If IsCallable(callback) is false, throw a TypeError exception.
-        // See: http://es5.github.com/#x9.11
-        if (typeof callback !== "function") {
-          throw new TypeError(callback + ' is not a function');
-        }
+            // 4. If IsCallable(callback) is false, throw a TypeError exception.
+            // See: http://es5.github.com/#x9.11
+            if (typeof callback !== "function") {
+                throw new TypeError(callback + ' is not a function');
+            }

-        // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
-        if (arguments.length > 1) {
-          T = thisArg;
-        }
+            // 5. If thisArg was supplied, let T be thisArg; else let T be undefined.
+            if (arguments.length > 1) {
+                T = thisArg;
+            }

-        // 6. Let k be 0
-        k = 0;
+            // 6. Let k be 0
+            k = 0;

-        // 7. Repeat, while k < len
-        while (k < len) {
-          let kValue;
+            // 7. Repeat, while k < len
+            while (k < len) {
+                let kValue;

-          // a. Let Pk be ToString(k).
-          //   This is implicit for LHS operands of the in operator
-          // b. Let kPresent be the result of calling the HasProperty internal
-          //    method of O with
-          //    argument Pk.
-          //   This step can be combined with c
-          // c. If kPresent is true, then
-          if (k in O) {
-            // i. Let kValue be the result of calling the Get internal method of O with
-            // argument Pk
-            kValue = O[k];
+                // a. Let Pk be ToString(k).
+                //   This is implicit for LHS operands of the in operator
+                // b. Let kPresent be the result of calling the HasProperty internal
+                //    method of O with
+                //    argument Pk.
+                //   This step can be combined with c
+                // c. If kPresent is true, then
+                if (k in O) {
+                    // i. Let kValue be the result of calling the Get internal method of O with
+                    // argument Pk
+                    kValue = O[k];

-            // ii. Call the Call internal method of callback with T as the this value and
-            // argument list containing kValue, k, and O.
-            callback.call(T, kValue, k, O);
-          }
-          // d. Increase k by 1.
-          k++;
-        }
+                    // ii. Call the Call internal method of callback with T as the this value and
+                    // argument list containing kValue, k, and O.
+                    callback.call(T, kValue, k, O);
+                }
+                // d. Increase k by 1.
+                k++;
+            }
        // 8. return undefined
-      };
+        };
    }
 }

@@ -655,7 +661,10 @@ export function isNumber(value: any): boolean {
 * @return {string} a string with the hidden characters removed
 */
 export function removeHiddenChars(str: string): string {
-    return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, ''));
+    if (typeof str === "string") {
+        return unhomoglyph(str.normalize('NFD').replace(removeHiddenCharsRegex, ''));
+    }
+    return "";
 }

 // Regex matching bunch of unicode control characters and otherwise misleading/invisible characters.
@@ -665,6 +674,7 @@ export function removeHiddenChars(str: string): string {
 // LTR/RTL and other directional formatting marks U+202A - U+202F
 // Combining characters U+0300 - U+036F
 // Zero width no-break space (BOM) U+FEFF
+// eslint-disable-next-line no-misleading-character-class
 const removeHiddenCharsRegex = /[\u2000-\u200F\u202A-\u202F\u0300-\u036f\uFEFF\s]/g;

 export function escapeRegExp(string: string): string {
@@ -723,9 +733,9 @@ export function defer() {
 }

 export async function promiseMapSeries<T>(
-        promises: Promise<T>[],
-        fn: (t: T) => void
-    ): Promise<void> {
+    promises: Promise<T>[],
+    fn: (t: T) => void,
+): Promise<void> {
    for (const o of await promises) {
        await fn(await o);
    }
@@ -165,29 +165,26 @@ MatrixCall.prototype.placeVideoCall = function(remoteVideoElement, localVideoEle
 * @throws If you have not specified a listener for 'error' events.
 */
 MatrixCall.prototype.placeScreenSharingCall =
-    function(remoteVideoElement, localVideoElement) {
+    async function(remoteVideoElement, localVideoElement) {
    debuglog("placeScreenSharingCall");
    checkForErrorListener(this);
-    const screenConstraints = _getScreenSharingConstraints(this);
-    if (!screenConstraints) {
-        return;
-    }
    this.localVideoElement = localVideoElement;
    this.remoteVideoElement = remoteVideoElement;
    const self = this;
-    this.webRtc.getUserMedia(screenConstraints, function(stream) {
-        self.screenSharingStream = stream;
+    try {
+        self.screenSharingStream = await this.webRtc.getDisplayMedia({'audio': false});
        debuglog("Got screen stream, requesting audio stream...");
        const audioConstraints = _getUserMediaVideoContraints('voice');
        _placeCallWithConstraints(self, audioConstraints);
-    }, function(err) {
+    } catch (err) {
        self.emit("error",
            callError(
                MatrixCall.ERR_NO_USER_MEDIA,
                "Failed to get screen-sharing stream: " + err,
            ),
        );
-    });
+    }
+
    this.type = 'video';
    _tryPlayRemoteStream(this);
 };
@@ -406,7 +403,7 @@ MatrixCall.prototype._initWithHangup = function(event) {
 * Answer a call.
 */
 MatrixCall.prototype.answer = function() {
-    debuglog("Answering call %s of type %s", this.callId, this.type);
+    debuglog(`Answering call ${this.callId} of type ${this.type}`);
    const self = this;

    if (self._answerContent) {
@@ -415,8 +412,10 @@ MatrixCall.prototype.answer = function() {
    }

    if (!this.localAVStream && !this.waitForLocalAVStream) {
+        const constraints = _getUserMediaVideoContraints(this.type);
+        logger.log("Getting user media with constraints", constraints);
        this.webRtc.getUserMedia(
-            _getUserMediaVideoContraints(this.type),
+            constraints,
            hookCallback(self, self._maybeGotUserMediaForAnswer),
            hookCallback(self, self._maybeGotUserMediaForAnswer),
        );
@@ -1068,7 +1067,7 @@ const terminate = function(self, hangupParty, hangupReason, shouldEmit) {
 };

 const stopAllMedia = function(self) {
-    debuglog("stopAllMedia (stream=%s)", self.localAVStream);
+    debuglog(`stopAllMedia (stream=${self.localAVStream})`);
    if (self.localAVStream) {
        forAllTracksOnStream(self.localAVStream, function(t) {
            if (t.stop) {
@@ -1130,7 +1129,11 @@ const _tryPlayRemoteAudioStream = async function(self) {
        const player = self.getRemoteAudioElement();

        // if audioOutput is non-default:
-        if (audioOutput) await player.setSinkId(audioOutput);
+        try {
+            if (audioOutput) await player.setSinkId(audioOutput);
+        } catch (e) {
+            logger.warn("Couldn't set requested audio output device: using default", e);
+        }

        player.autoplay = true;
        self.assignElement(player, self.remoteAStream, "remoteAudio");
@@ -1191,8 +1194,8 @@ const _sendCandidateQueue = function(self) {

        if (self.candidateSendTries > 5) {
            debuglog(
-                "Failed to send candidates on attempt %s. Giving up for now.",
-                self.candidateSendTries,
+                "Failed to send candidates on attempt " + self.candidateSendTries +
+                ". Giving up for now.",
            );
            self.candidateSendTries = 0;
            return;
@@ -1208,6 +1211,7 @@ const _sendCandidateQueue = function(self) {
 };

 const _placeCallWithConstraints = function(self, constraints) {
+    logger.log("Getting user media with constraints", constraints);
    self.client.callList[self.callId] = self;
    self.webRtc.getUserMedia(
        constraints,
@@ -1231,31 +1235,6 @@ const _createPeerConnection = function(self) {
    return pc;
 };

-const _getScreenSharingConstraints = function(call) {
-    const screen = global.screen;
-    if (!screen) {
-        call.emit("error", callError(
-            MatrixCall.ERR_NO_USER_MEDIA,
-            "Couldn't determine screen sharing constaints.",
-        ));
-        return;
-    }
-
-    return {
-        video: {
-            mediaSource: 'screen',
-            mandatory: {
-                chromeMediaSource: "screen",
-                chromeMediaSourceId: "" + Date.now(),
-                maxWidth: screen.width,
-                maxHeight: screen.height,
-                minFrameRate: 1,
-                maxFrameRate: 10,
-            },
-        },
-    };
-};
-
 const _getUserMediaVideoContraints = function(callType) {
    const isWebkit = !!global.window.navigator.webkitGetUserMedia;

@@ -1371,6 +1350,14 @@ export function createNewMatrixCall(client, roomId, options) {
        };
    }

+    const getDisplayMedia = (
+        w.navigator.mediaDevices && w.navigator.mediaDevices.getDisplayMedia ||
+        w.navigator.getDisplayMedia
+    );
+    if (getDisplayMedia) {
+        webRtc.getDisplayMedia = getDisplayMedia.bind(w.navigator.mediaDevices);
+    }
+
    // Firefox throws on so little as accessing the RTCPeerConnection when operating in
    // a secure mode. There's some information at https://bugzilla.mozilla.org/show_bug.cgi?id=1542616
    // though the concern is that the browser throwing a SecurityError will brick the
@@ -2,6 +2,7 @@
  "compilerOptions": {
    "experimentalDecorators": true,
    "emitDecoratorMetadata": true,
+    "esModuleInterop": true,
    "module": "commonjs",
    "moduleResolution": "node",
    "target": "es2016",
@@ -1,72 +0,0 @@
-{
-  "rules": {
-    "class-name": false,
-    "comment-format": [
-      true
-    ],
-    "curly": false,
-    "eofline": false,
-    "forin": false,
-    "indent": [
-      true,
-      "spaces"
-    ],
-    "label-position": true,
-    "max-line-length": false,
-    "member-access": false,
-    "member-ordering": [
-      true,
-      "static-after-instance",
-      "variables-before-functions"
-    ],
-    "no-arg": true,
-    "no-bitwise": false,
-    "no-console": false,
-    "no-construct": true,
-    "no-debugger": true,
-    "no-duplicate-variable": true,
-    "no-empty": false,
-    "no-eval": true,
-    "no-inferrable-types": true,
-    "no-shadowed-variable": true,
-    "no-string-literal": false,
-    "no-switch-case-fall-through": true,
-    "no-trailing-whitespace": true,
-    "no-unused-expression": true,
-    "no-use-before-declare": false,
-    "no-var-keyword": true,
-    "object-literal-sort-keys": false,
-    "one-line": [
-      true,
-      "check-open-brace",
-      "check-catch",
-      "check-else",
-      "check-whitespace"
-    ],
-    "quotemark": false,
-    "radix": true,
-    "semicolon": [
-      "always"
-    ],
-    "triple-equals": [],
-    "typedef-whitespace": [
-      true,
-      {
-        "call-signature": "nospace",
-        "index-signature": "nospace",
-        "parameter": "nospace",
-        "property-declaration": "nospace",
-        "variable-declaration": "nospace"
-      }
-    ],
-    "variable-name": false,
-    "whitespace": [
-      true,
-      "check-branch",
-      "check-decl",
-      "check-operator",
-      "check-separator",
-      "check-type"
-    ]
-  }
-}