v30.3.0

* Comment explaining the purpose of RoomList

* Fix incorrect return type declaration on RoomList.getRoomEncryption

* Move RoomList out of MatrixClient, into legacy Crypto

* Initialise RoomList inside Crypto.init to allow us to await it

.eslintrc.js

@@ -66,9 +66,6 @@ module.exports = {
         // Disabled tests are a reality for now but as soon as all of the xits are
         // eliminated, we should enforce this.
         "jest/no-disabled-tests": "off",
-        // TODO: There are many tests with invalid expects that should be fixed,
-        // https://github.com/matrix-org/matrix-js-sdk/issues/2976
-        "jest/valid-expect": "off",
         // Also treat "oldBackendOnly" as a test function.
         // Used in some crypto tests.
         "jest/no-standalone-expect": [

.github/CODEOWNERS

@@ -3,4 +3,6 @@
 /package.json             @matrix-org/element-web-app-team
 /yarn.lock                @matrix-org/element-web-app-team
 /src/webrtc               @matrix-org/element-call-reviewers
+/src/matrixrtc               @matrix-org/element-call-reviewers
 /spec/*/webrtc            @matrix-org/element-call-reviewers
+/spec/*/matrixrtc            @matrix-org/element-call-reviewers

.github/actions/sign-release-tarball/action.yml

@@ -0,0 +1,28 @@
+name: Sign Release Tarball
+description: Generates signature for release tarball and uploads it as a release asset
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the tarball.
+        required: true
+    upload-url:
+        description: GitHub release upload URL to upload the signature file to.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Generate tarball signature
+          shell: bash
+          run: |
+              git -c tar.tar.gz.command='gzip -cn' archive --format=tar.gz --prefix="${REPO#*/}-${VERSION#v}/" -o "/tmp/${VERSION}.tar.gz" "${VERSION}"
+              gpg -u "$GPG_FINGERPRINT" --armor --output "${VERSION}.tar.gz.asc" --detach-sig "/tmp/${VERSION}.tar.gz"
+              rm "/tmp/${VERSION}.tar.gz"
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              REPO: ${{ github.repository }}
+
+        - name: Upload tarball signature
+          if: ${{ inputs.upload-url }}
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ env.VERSION }}.tar.gz.asc

.github/actions/upload-release-assets/action.yml

@@ -0,0 +1,41 @@
+name: Upload release assets
+description: Uploads assets to an existing release and optionally signs them
+inputs:
+    gpg-fingerprint:
+        description: Fingerprint of the GPG key to use for signing the assets, if any.
+        required: false
+    upload-url:
+        description: GitHub release upload URL to upload the assets to.
+        required: true
+    asset-path:
+        description: |
+            The path to the asset you want to upload, if any. You can use glob patterns here.
+            Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+        required: true
+runs:
+    using: composite
+    steps:
+        - name: Sign assets
+          if: inputs.gpg-fingerprint
+          shell: bash
+          run: |
+              for FILE in $ASSET_PATH
+              do
+                  gpg -u "$GPG_FINGERPRINT" --armor --output "$FILE".asc --detach-sig "$FILE"
+              done
+          env:
+              GPG_FINGERPRINT: ${{ inputs.gpg-fingerprint }}
+              ASSET_PATH: ${{ inputs.asset-path }}
+
+        - name: Upload asset signatures
+          if: inputs.gpg-fingerprint
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}.asc
+
+        - name: Upload assets
+          uses: shogo82148/actions-upload-release-asset@dccd6d23e64fd6a746dce6814c0bde0a04886085 # v1
+          with:
+              upload_url: ${{ inputs.upload-url }}
+              asset_path: ${{ inputs.asset-path }}

.github/release-drafter.yml

@@ -0,0 +1,31 @@
+name-template: "v$RESOLVED_VERSION"
+tag-template: "v$RESOLVED_VERSION"
+change-template: "* $TITLE ([#$NUMBER]($URL)). Contributed by @$AUTHOR."
+categories:
+    - title: "🚨 BREAKING CHANGES"
+      label: "X-Breaking-Change"
+    - title: "🦖 Deprecations"
+      label: "T-Deprecation"
+    - title: "✨ Features"
+      label: "T-Enhancement"
+    - title: "🐛 Bug Fixes"
+      label: "T-Defect"
+    - title: "🧰 Maintenance"
+      label: "Dependencies"
+      collapse-after: 5
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+version-resolver:
+    major:
+        labels:
+            - "X-Breaking-Change"
+    default: minor
+exclude-labels:
+    - "T-Task"
+    - "X-Reverted"
+exclude-contributors:
+    - "RiotRobot"
+template: |
+    $CHANGES
+prerelease: true
+prerelease-identifier: rc
+include-pre-releases: false

.github/workflows/backport.yml

@@ -23,7 +23,7 @@ jobs:
               )
             )
         steps:
-            - uses: tibdex/backport@2e217641d82d02ba0603f46b1aeedefb258890ac # v2
+            - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # v2
               with:
                   labels_template: "<%= JSON.stringify([...labels, 'X-Release-Blocker']) %>"
                   # We can't use GITHUB_TOKEN here or CI won't run on the new PR

.github/workflows/cypress.yml

@@ -1,7 +1,7 @@
 # Triggers after the "Downstream artifacts" build has finished, to run the
-# cypress tests (with access to repo secrets)
+# matrix-react-sdk playwright & cypress tests (with access to repo secrets)
 
-name: matrix-react-sdk Cypress End to End Tests
+name: matrix-react-sdk End to End Tests
 on:
     workflow_run:
         workflows: ["Build downstream artifacts"]
@@ -15,7 +15,12 @@ concurrency:
 jobs:
     cypress:
         name: Cypress
-        uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@v3.73.1
+
+        # We only want to run the cypress tests on merge queue to prevent regressions
+        # from creeping in. They take a long time to run and consume multiple concurrent runners.
+        if: github.event.workflow_run.event == 'merge_group'
+
+        uses: matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml@develop
         permissions:
             actions: read
             issues: read
@@ -23,9 +28,63 @@ jobs:
             pull-requests: read
         secrets:
             # secrets are not automatically shared with called workflows, so share the cypress dashboard key, and the Kiwi login details
-            CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+            KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_RUST: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_RUST}}
+            KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_LEGACY: ${{ secrets.KNAPSACK_PRO_TEST_SUITE_TOKEN_CYPRESS_LEGACY}}
             TCMS_USERNAME: ${{ secrets.TCMS_USERNAME }}
             TCMS_PASSWORD: ${{ secrets.TCMS_PASSWORD }}
         with:
             react-sdk-repository: matrix-org/matrix-react-sdk
-            rust-crypto: true
+
+    playwright:
+        name: Playwright
+        # We only want to run the playwright tests on merge queue to prevent regressions
+        # from creeping in. They take a long time to run and consume multiple concurrent runners.
+        if: github.event.workflow_run.event == 'merge_group'
+        uses: matrix-org/matrix-react-sdk/.github/workflows/end-to-end-tests.yaml@develop
+        permissions:
+            actions: read
+            issues: read
+            statuses: write
+            pull-requests: read
+            deployments: write
+        with:
+            react-sdk-repository: matrix-org/matrix-react-sdk
+
+    # We want to make the cypress tests a required check for the merge queue.
+    #
+    # Unfortunately, github doesn't distinguish between "checks needed for branch
+    # protection" (ie, the things that must pass before the PR will even be added
+    # to the merge queue) and "checks needed in the merge queue". We just have to add
+    # the check to the branch protection list.
+    #
+    # Ergo, if we know we're not going to run the cypress tests, we need to add a
+    # passing status check manually.
+    mark_skipped:
+        if: github.event.workflow_run.event != 'merge_group'
+        permissions:
+            statuses: write
+        runs-on: ubuntu-latest
+        steps:
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
+              with:
+                  authToken: "${{ secrets.GITHUB_TOKEN }}"
+                  state: success
+                  description: Cypress skipped
+
+                  # Keep in step with the `context` that is updated by `Sibz/github-status-action`
+                  # in matrix-org/matrix-react-sdk/.github/workflows/cypress.yaml.
+                  context: "${{ github.workflow }} / cypress"
+
+                  sha: "${{ github.event.workflow_run.head_sha }}"
+
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
+              with:
+                  authToken: "${{ secrets.GITHUB_TOKEN }}"
+                  state: success
+                  description: Playwright skipped
+
+                  # Keep in step with the `context` that is updated by `Sibz/github-status-action`
+                  # in matrix-org/matrix-react-sdk/.github/workflows/end-to-end-tests.yaml.
+                  context: "${{ github.workflow }} / end-to-end-tests"
+
+                  sha: "${{ github.event.workflow_run.head_sha }}"

.github/workflows/docs-pr-netlify.yaml

@@ -22,7 +22,7 @@ jobs:
                   path: docs
 
             - name: 📤 Deploy to Netlify
-              uses: matrix-org/netlify-pr-preview@v2
+              uses: matrix-org/netlify-pr-preview@v3
               with:
                   path: docs
                   owner: ${{ github.event.workflow_run.head_repository.owner.login }}
@@ -32,3 +32,4 @@ jobs:
                   site_id: ${{ secrets.NETLIFY_SITE_ID }}
                   desc: Documentation preview
                   deployment_env: PR Documentation Preview
+        environment: PR Documentation Preview

.github/workflows/downstream-artifacts.yml

@@ -1,9 +1,10 @@
 name: Build downstream artifacts
 on:
-    pull_request: {}
     merge_group:
         types: [checks_requested]
 
+    pull_request: {}
+
     # For now at least, we don't run this or the cypress-tests against pushes
     # to develop or master.
     #
@@ -19,7 +20,7 @@ concurrency:
 jobs:
     build-element-web:
         name: Build element-web
-        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.73.1
+        uses: matrix-org/matrix-react-sdk/.github/workflows/element-web.yaml@v3.85.0
         with:
             matrix-js-sdk-sha: ${{ github.sha }}
             react-sdk-repository: matrix-org/matrix-react-sdk

.github/workflows/notify-downstream.yaml

@@ -20,7 +20,7 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: Notify matrix-react-sdk repo that a new SDK build is on develop so it can CI against it
-              uses: peter-evans/repository-dispatch@26b39ed245ab8f31526069329e112ab2fb224588 # v2
+              uses: peter-evans/repository-dispatch@bf47d102fdb849e755b0b0023ea3e81a44b6f570 # v2
               with:
                   token: ${{ secrets.ELEMENT_BOT_TOKEN }}
                   repository: ${{ matrix.repo }}

.github/workflows/pull_request.yaml

@@ -27,7 +27,7 @@ jobs:
             pull-requests: read
         steps:
             - name: Add notice
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               if: contains(github.event.pull_request.labels.*.name, 'X-Blocked')
               with:
                   script: |
@@ -39,7 +39,7 @@ jobs:
         if: github.event.action == 'opened'
         steps:
             - name: Check membership
-              uses: tspascoal/get-user-teams-membership@37c08f7b52a72ca95d12af2e7ab2553ca9adf13b # v2
+              uses: tspascoal/get-user-teams-membership@ba78054988f58bea69b7c6136d563236f8ed2fc0 # v3
               id: teams
               with:
                   username: ${{ github.event.pull_request.user.login }}
@@ -49,7 +49,7 @@ jobs:
 
             - name: Add label
               if: ${{ steps.teams.outputs.isTeamMember == 'false' }}
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               with:
                   script: |
                       github.rest.issues.addLabels({
@@ -68,7 +68,7 @@ jobs:
             github.event.pull_request.head.repo.full_name != github.repository
         steps:
             - name: Close pull request
-              uses: actions/github-script@v6
+              uses: actions/github-script@v7
               with:
                   script: |
                       github.rest.issues.createComment({

.github/workflows/release-drafter.yml

@@ -0,0 +1,21 @@
+name: Release Drafter
+on:
+    push:
+        branches: [staging]
+    workflow_dispatch:
+        inputs:
+            previous-version:
+                description: What release to use as a base for release note purposes
+                required: false
+                type: string
+concurrency: ${{ github.workflow }}
+jobs:
+    draft:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: release-drafter/release-drafter@e64b19c4c46173209ed9f2e5a2f4ca7de89a0e86 # v5
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                  disable-autolabeler: true
+                  previous-version: ${{ inputs.previous-version }}

.github/workflows/release-gitflow.yml

@@ -0,0 +1,85 @@
+# Gitflow merge-back master->develop
+name: Merge master -> develop
+on:
+    push:
+        branches: [master]
+    workflow_call:
+        secrets:
+            ELEMENT_BOT_TOKEN:
+                required: true
+        inputs:
+            dependencies:
+                description: List of dependencies to reset.
+                type: string
+                required: false
+concurrency: ${{ github.workflow }}
+jobs:
+    merge:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+              with:
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      scripts/release
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Set up git
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+
+            - name: Merge to develop
+              run: |
+                  git checkout develop
+                  git merge -X ours master
+
+            - name: Run post-merge-master script to revert package.json fields
+              run: ./.action-repo/scripts/release/post-merge-master.sh
+
+            - name: Reset dependencies
+              if: inputs.dependencies
+              run: |
+                  while IFS= read -r PACKAGE; do
+                      [ -z "$PACKAGE" ] && continue
+
+                      CURRENT_VERSION=$(cat package.json | jq -r .dependencies[\"$PACKAGE\"])
+                      echo "Current $PACKAGE version is $CURRENT_VERSION"
+
+                      if [ "$CURRENT_VERSION" == "null" ]
+                      then
+                          echo "Unable to find $PACKAGE in package.json"
+                          exit 1
+                      fi
+
+                      if [ "$CURRENT_VERSION" == "develop" ]
+                      then
+                          echo "Not updating dependency $PACKAGE"
+                          continue
+                      fi
+
+                      echo "Resetting $1 to develop branch..."
+                      yarn add "github:matrix-org/$PACKAGE#develop"
+                      git add -u
+                      git commit -m "Reset $PACKAGE back to develop branch"
+                  done <<< "$DEPENDENCIES"
+              env:
+                  DEPENDENCIES: ${{ inputs.dependencies }}
+                  FINAL: ${{ inputs.final }}
+
+            - name: Push changes
+              run: git push origin develop

.github/workflows/release-make.yml

@@ -0,0 +1,353 @@
+name: Release Make
+on:
+    workflow_call:
+        secrets:
+            ELEMENT_BOT_TOKEN:
+                required: true
+            NPM_TOKEN:
+                required: false
+            GPG_PASSPHRASE:
+                required: false
+            GPG_PRIVATE_KEY:
+                required: false
+        inputs:
+            final:
+                description: Make final release
+                required: true
+                default: false
+                type: boolean
+            npm:
+                description: Publish to npm
+                type: boolean
+                default: false
+            dependencies:
+                description: |
+                    List of dependencies to update in `npm-dep=version` format.
+                    `version` can be `"current"` to leave it at the current version.
+                type: string
+                required: false
+            include-changes:
+                description: Project to include changelog entries from in this release.
+                type: string
+                required: false
+            gpg-fingerprint:
+                description: Fingerprint of the GPG key to use for signing the git tag and assets, if any.
+                type: string
+                required: false
+            asset-path:
+                description: |
+                    The path to the asset you want to upload, if any. You can use glob patterns here.
+                    Will be GPG signed and an `.asc` file included in the release artifacts if `gpg-fingerprint` is set.
+                type: string
+                required: false
+            expected-asset-count:
+                description: The number of expected assets, including signatures, excluding generated zip & tarball.
+                type: number
+                required: false
+jobs:
+    release:
+        name: Release
+        runs-on: ubuntu-latest
+        environment: Release
+        steps:
+            - name: Load GPG key
+              id: gpg
+              if: inputs.gpg-fingerprint
+              uses: crazy-max/ghaction-import-gpg@82a020f1f7f605c65dd2449b392a52c3fcfef7ef # v6
+              with:
+                  gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
+                  passphrase: ${{ secrets.GPG_PASSPHRASE }}
+                  fingerprint: ${{ inputs.gpg-fingerprint }}
+
+            - name: Get draft release
+              id: release
+              uses: cardinalby/git-get-release-action@cedef2faf69cb7c55b285bad07688d04430b7ada # v1
+              env:
+                  GITHUB_TOKEN: ${{ github.token }}
+              with:
+                  draft: true
+                  latest: true
+
+            - uses: actions/checkout@v4
+              with:
+                  ref: staging
+                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  fetch-depth: 0
+
+            - name: Get actions scripts
+              uses: actions/checkout@v4
+              with:
+                  repository: matrix-org/matrix-js-sdk
+                  persist-credentials: false
+                  path: .action-repo
+                  sparse-checkout: |
+                      .github/actions
+                      scripts/release
+
+            - name: Prepare variables
+              id: prepare
+              run: |
+                  echo "VERSION=$VERSION" >> $GITHUB_ENV
+                  {
+                      echo "RELEASE_NOTES<<EOF"
+                      echo "$BODY"
+                      echo "EOF"
+                  } >> $GITHUB_ENV
+
+                  HAS_DIST=0
+                  jq -e .scripts.dist package.json >/dev/null 2>&1 && HAS_DIST=1
+                  echo "has-dist-script=$HAS_DIST" >> $GITHUB_OUTPUT
+              env:
+                  BODY: ${{ steps.release.outputs.body }}
+                  VERSION: ${{ steps.release.outputs.tag_name }}
+
+            - name: Finalise version
+              if: inputs.final
+              run: echo "VERSION=$(echo $VERSION | cut -d- -f1)" >> $GITHUB_ENV
+
+            - name: Check version number not in use
+              uses: actions/github-script@v7
+              with:
+                  script: |
+                      const { VERSION } = process.env;
+                      github.rest.repos.getReleaseByTag({
+                          owner: context.repo.owner,
+                          repo: context.repo.repo,
+                          tag: VERSION,
+                      }).then(() => {
+                          core.setFailed(`Version ${VERSION} already exists`);
+                      }).catch(() => {
+                        // This is fine, we expect there to not be any release with this version yet
+                      });
+
+            - name: Set up git
+              run: |
+                  git config --global user.email "releases@riot.im"
+                  git config --global user.name "RiotRobot"
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install dependencies
+              run: "yarn install --frozen-lockfile"
+
+            - name: Update dependencies
+              id: update-dependencies
+              if: inputs.dependencies
+              run: |
+                  UPDATED=()
+                  while IFS= read -r DEPENDENCY; do
+                      [ -z "$DEPENDENCY" ] && continue
+                      IFS="=" read -r PACKAGE UPDATE_VERSION <<< "$DEPENDENCY"
+
+                      CURRENT_VERSION=$(cat package.json | jq -r .dependencies[\"$PACKAGE\"])
+                      echo "Current $PACKAGE version is $CURRENT_VERSION"
+
+                      if [ "$CURRENT_VERSION" == "null" ]
+                      then
+                          echo "Unable to find $PACKAGE in package.json"
+                          exit 1
+                      fi
+
+                      if [ "$UPDATE_VERSION" == "current" ] || [ "$UPDATE_VERSION" == "$CURRENT_VERSION" ]
+                      then
+                          echo "Not updating dependency $PACKAGE"
+                          continue
+                      fi
+
+                      echo "Upgrading $PACKAGE to $UPDATE_VERSION..."
+                      yarn upgrade "$PACKAGE@$UPDATE_VERSION" --exact
+                      git add -u
+                      git commit -m "Upgrade $PACKAGE to $UPDATE_VERSION"
+                      UPDATED+=("$PACKAGE")
+                  done <<< "$DEPENDENCIES"
+
+                  JSON=$(jq --compact-output --null-input '$ARGS.positional' --args -- "${UPDATED[@]}")
+                  echo "updated=$JSON" >> $GITHUB_OUTPUT
+              env:
+                  DEPENDENCIES: ${{ inputs.dependencies }}
+
+            - name: Prevent develop dependencies
+              if: inputs.dependencies
+              run: |
+                  ret=0
+                  cat package.json | jq '.dependencies[]' | grep -q '#develop' || ret=$?
+                  if [ "$ret" -eq 0 ]; then
+                      echo "package.json contains develop dependencies. Refusing to release."
+                      exit
+                  fi
+
+            - name: Bump package.json version
+              run: yarn version --no-git-tag-version --new-version "${VERSION#v}"
+
+            - name: Ingest upstream changes
+              if: |
+                  inputs.include-changes &&
+                  (!inputs.dependencies || contains(fromJSON(steps.update-dependencies.outputs.updated), inputs.include-changes))
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  DEPENDENCY: ${{ inputs.include-changes }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: releaseId, DEPENDENCY, VERSION } = process.env;
+                      const { owner, repo } = context.repo;
+                      const script = require("./.action-repo/scripts/release/merge-release-notes.js");
+                      const notes = await script({
+                          github,
+                          releaseId,
+                          dependencies: [DEPENDENCY.replace("$VERSION", VERSION)],
+                      });
+                      core.exportVariable("RELEASE_NOTES", notes);
+
+            - name: Add to CHANGELOG.md
+              if: inputs.final
+              run: |
+                  mv CHANGELOG.md CHANGELOG.md.old
+                  HEADER="Changes in [${VERSION#v}](https://github.com/${{ github.repository }}/releases/tag/$VERSION) ($(date '+%Y-%m-%d'))"
+
+                  {
+                      echo "$HEADER"
+                      printf '=%.0s' $(seq ${#HEADER})
+                      echo ""
+                      echo "$RELEASE_NOTES"
+                      echo ""
+                  } > CHANGELOG.md
+
+                  cat CHANGELOG.md.old >> CHANGELOG.md
+                  rm CHANGELOG.md.old
+                  git add CHANGELOG.md
+
+            - name: Run pre-release script to update package.json fields
+              run: |
+                  ./.action-repo/scripts/release/pre-release.sh
+                  git add package.json
+
+            - name: Commit changes
+              run: git commit -m "$VERSION"
+
+            - name: Build assets
+              if: steps.prepare.outputs.has-dist-script == '1'
+              run: DIST_VERSION="$VERSION" yarn dist
+
+            - name: Upload release assets & signatures
+              if: inputs.asset-path
+              uses: ./.action-repo/.github/actions/upload-release-assets
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.release.outputs.upload_url }}
+                  asset-path: ${{ inputs.asset-path }}
+
+            - name: Create signed tag
+              if: inputs.gpg-fingerprint
+              run: |
+                  GIT_COMMITTER_EMAIL="$SIGNING_ID" GPG_TTY=$(tty) git tag -u "$SIGNING_ID" -m "Release $VERSION" "$VERSION"
+              env:
+                  SIGNING_ID: ${{ steps.gpg.outputs.email }}
+
+            - name: Generate & upload tarball signature
+              if: inputs.gpg-fingerprint
+              uses: ./.action-repo/.github/actions/sign-release-tarball
+              with:
+                  gpg-fingerprint: ${{ inputs.gpg-fingerprint }}
+                  upload-url: ${{ steps.release.outputs.upload_url }}
+
+            # We defer pushing changes until after the release assets are built,
+            # signed & uploaded to improve the atomicity of this action.
+            - name: Push changes to staging
+              run: |
+                  git push origin staging $TAG
+                  git reset --hard
+              env:
+                  TAG: ${{ inputs.gpg-fingerprint && env.VERSION || '' }}
+
+            - name: Validate tarball signature
+              if: inputs.gpg-fingerprint
+              run: |
+                  wget https://github.com/$GITHUB_REPOSITORY/archive/refs/tags/$VERSION.tar.gz
+                  gpg --verify "$VERSION.tar.gz.asc" "$VERSION.tar.gz"
+
+            - name: Validate release has expected assets
+              if: inputs.expected-asset-count
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  EXPECTED_ASSET_COUNT: ${{ inputs.expected-asset-count }}
+              with:
+                  retries: 3
+                  script: |
+                      const { RELEASE_ID: release_id, EXPECTED_ASSET_COUNT } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const { data: release } = await github.rest.repos.getRelease({
+                          owner,
+                          repo,
+                          release_id,
+                      });
+
+                      if (release.assets.length !== parseInt(EXPECTED_ASSET_COUNT, 10)) {
+                          core.setFailed(`Found ${release.assets.length} assets but expected ${EXPECTED_ASSET_COUNT}`);
+                      }
+
+            - name: Merge to master
+              if: inputs.final
+              run: |
+                  git checkout master
+                  git merge -X theirs staging
+                  git push origin master
+
+            - name: Publish release
+              uses: actions/github-script@v7
+              env:
+                  RELEASE_ID: ${{ steps.release.outputs.id }}
+                  FINAL: ${{ inputs.final }}
+              with:
+                  retries: 3
+                  github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
+                  script: |
+                      const { RELEASE_ID: release_id, RELEASE_NOTES, VERSION, FINAL } = process.env;
+                      const { owner, repo } = context.repo;
+
+                      const opts = {
+                          owner,
+                          repo,
+                          release_id,
+                          tag_name: VERSION,
+                          name: VERSION,
+                          draft: false,
+                          body: RELEASE_NOTES,
+                      };
+
+                      if (FINAL == "true") {
+                          opts.prerelease = false;
+                          opts.make_latest = true;
+                      }
+
+                      github.rest.repos.updateRelease(opts);
+
+    npm:
+        name: Publish to npm
+        needs: release
+        if: inputs.npm
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-npm.yml@develop
+        secrets:
+            NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+
+    update-labels:
+        name: Advance release blocker labels
+        needs: release
+        runs-on: ubuntu-latest
+        steps:
+            - id: repository
+              run: echo "REPO=${GITHUB_REPOSITORY#*/}" >> $GITHUB_OUTPUT
+
+            - uses: garganshu/github-label-updater@3770d15ebfed2fe2cb06a241047bc340f774a7d1 # v1.0.0
+              with:
+                  owner: ${{ github.repository_owner }}
+                  repo: ${{ steps.repository.outputs.REPO }}
+                  token: ${{ secrets.GITHUB_TOKEN }}
+                  filter-labels: X-Upcoming-Release-Blocker
+                  remove-labels: X-Upcoming-Release-Blocker
+                  add-labels: X-Release-Blocker

.github/workflows/release-npm.yml

@@ -1,4 +1,3 @@
-# Must only be called from `release#published` triggers
 name: Publish to npm
 on:
     workflow_call:
@@ -11,10 +10,12 @@ jobs:
         runs-on: ubuntu-latest
         steps:
             - name: 🧮 Checkout code
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
+              with:
+                  ref: staging
 
             - name: 🔧 Yarn cache
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
                   registry-url: "https://registry.npmjs.org"
@@ -24,7 +25,7 @@ jobs:
 
             - name: 🚀 Publish to npm
               id: npm-publish
-              uses: JS-DevTools/npm-publish@a25b4180b728b0279fca97d4e5bccf391685aead # v2.2.0
+              uses: JS-DevTools/npm-publish@4b07b26a2f6e0a51846e1870223e545bae91c552 # v3.0.1
               with:
                   token: ${{ secrets.NPM_TOKEN }}
                   access: public
@@ -32,7 +33,7 @@ jobs:
                   ignore-scripts: false
 
             - name: 🎖️ Add `latest` dist-tag to final releases
-              if: github.event.release.prerelease == false && steps.npm-publish.outputs.id
+              if: steps.npm-publish.outputs.id && !contains(steps.npm-publish.outputs.id, '-rc.')
               run: npm dist-tag add "$release" latest
               env:
                   NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/release.yml

@@ -1,24 +1,51 @@
 name: Release Process
 on:
-    release:
-        types: [published]
-concurrency: ${{ github.workflow }}-${{ github.ref }}
+    workflow_dispatch:
+        inputs:
+            mode:
+                description: What type of release
+                required: true
+                default: rc
+                type: choice
+                options:
+                    - rc
+                    - final
+            docs:
+                description: Publish docs
+                required: true
+                type: boolean
+                default: true
+            npm:
+                description: Publish to npm
+                required: true
+                type: boolean
+                default: true
+concurrency: ${{ github.workflow }}
 jobs:
-    jsdoc:
+    release:
+        uses: matrix-org/matrix-js-sdk/.github/workflows/release-make.yml@develop
+        secrets: inherit
+        with:
+            final: ${{ inputs.mode == 'final' }}
+            npm: ${{ inputs.npm }}
+
+    docs:
         name: Publish Documentation
+        needs: release
+        if: inputs.docs
         runs-on: ubuntu-latest
         steps:
             - name: 🧮 Checkout code
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
 
             - name: 🧮 Checkout gh-pages
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
               with:
                   ref: gh-pages
                   path: _docs
 
             - name: 🔧 Yarn cache
-              uses: actions/setup-node@v3
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -36,17 +63,14 @@ jobs:
                   yarn gendoc
                   symlinks -rc _docs
 
-            - name: 🚀 Deploy
+            - name: 🔨 Set up git
               run: |
                   git config --global user.email "releases@riot.im"
                   git config --global user.name "RiotRobot"
+
+            - name: 🚀 Deploy
+              run: |
                   git add . --all
                   git commit -m "Update docs"
                   git push
               working-directory: _docs
-
-    npm:
-        name: Publish
-        uses: matrix-org/matrix-js-sdk/.github/workflows/release-npm.yml@develop
-        secrets:
-            NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/sonarcloud.yml

@@ -17,7 +17,7 @@ jobs:
         steps:
             # We create the status here and then update it to success/failure in the `report` stage
             # This provides an easy link to this workflow_run from the PR before Cypress is done.
-            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
                   state: pending
@@ -27,7 +27,7 @@ jobs:
 
             - name: "🩻 SonarCloud Scan"
               id: sonarcloud
-              uses: matrix-org/sonarcloud-workflow-action@v2.5
+              uses: matrix-org/sonarcloud-workflow-action@v2.7
               # workflow_run fails report against the develop commit always, we don't want that for PRs
               continue-on-error: ${{ github.event.workflow_run.head_branch != 'develop' }}
               with:
@@ -42,7 +42,7 @@ jobs:
                   coverage_extract_path: coverage
                   extra_args: ${{ inputs.extra_args }}
 
-            - uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+            - uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               if: always()
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/static_analysis.yml

@@ -13,9 +13,9 @@ jobs:
         name: "Typescript Syntax Check"
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -39,9 +39,9 @@ jobs:
         name: "ESLint"
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 
@@ -51,13 +51,29 @@ jobs:
             - name: Run Linter
               run: "yarn run lint:js"
 
+    workflow_lint:
+        name: "Workflow Lint"
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v4
+
+            - uses: actions/setup-node@v4
+              with:
+                  cache: "yarn"
+
+            - name: Install Deps
+              run: "yarn install --frozen-lockfile"
+
+            - name: Run Linter
+              run: "yarn lint:workflows"
+
     docs:
         name: "JSDoc Checker"
         runs-on: ubuntu-latest
         steps:
-            - uses: actions/checkout@v3
+            - uses: actions/checkout@v4
 
-            - uses: actions/setup-node@v3
+            - uses: actions/setup-node@v4
               with:
                   cache: "yarn"
 

.github/workflows/tests.yml

@@ -12,19 +12,20 @@ env:
     ENABLE_COVERAGE: ${{ github.event_name != 'merge_group' }}
 jobs:
     jest:
-        name: "Jest [${{ matrix.specs }}] (Node ${{ matrix.node }})"
+        name: "Jest [${{ matrix.specs }}] (Node ${{ matrix.node == '*' && 'latest' || matrix.node }})"
         runs-on: ubuntu-latest
         timeout-minutes: 10
         strategy:
             matrix:
-                specs: [browserify, integ, unit]
-                node: [16, 18, latest]
+                specs: [integ, unit]
+                node: [18, "lts/*", 21]
         steps:
             - name: Checkout code
-              uses: actions/checkout@v3
+              uses: actions/checkout@v4
 
             - name: Setup Node
-              uses: actions/setup-node@v3
+              id: setupNode
+              uses: actions/setup-node@v4
               with:
                   cache: "yarn"
                   node-version: ${{ matrix.node }}
@@ -32,13 +33,9 @@ jobs:
             - name: Install dependencies
               run: "yarn install"
 
-            - name: Build
-              if: matrix.specs == 'browserify'
-              run: "yarn build"
-
             - name: Get number of CPU cores
               id: cpu-cores
-              uses: SimenB/github-actions-cpu-cores@410541432439795d30db6501fb1d8178eb41e502 # v1
+              uses: SimenB/github-actions-cpu-cores@97ba232459a8e02ff6121db9362b09661c875ab8 # v2
 
             - name: Run tests
               run: |
@@ -50,9 +47,12 @@ jobs:
               env:
                   JEST_SONAR_UNIQUE_OUTPUT_NAME: true
 
+                  # tell jest to use coloured output
+                  FORCE_COLOR: true
+
             - name: Move coverage files into place
               if: env.ENABLE_COVERAGE == 'true'
-              run: mv coverage/lcov.info coverage/${{ matrix.node }}-${{ matrix.specs }}.lcov.info
+              run: mv coverage/lcov.info coverage/${{ steps.setupNode.output.node-version }}-${{ matrix.specs }}.lcov.info
 
             - name: Upload Artifact
               if: env.ENABLE_COVERAGE == 'true'
@@ -82,7 +82,7 @@ jobs:
         steps:
             - name: Skip SonarCloud on merge queues
               if: env.ENABLE_COVERAGE == 'false'
-              uses: Sibz/github-status-action@faaa4d96fecf273bd762985e0e7f9f933c774918 # v1
+              uses: Sibz/github-status-action@071b5370da85afbb16637d6eed8524a06bc2053e # v1
               with:
                   authToken: ${{ secrets.GITHUB_TOKEN }}
                   state: success

.github/workflows/triage-labelled.yml

@@ -0,0 +1,11 @@
+name: Move labelled issues to correct projects
+
+on:
+    issues:
+        types: [labeled]
+
+jobs:
+    call-triage-labelled:
+        uses: vector-im/element-web/.github/workflows/triage-labelled.yml@develop
+        secrets:
+            ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}

.github/workflows/upgrade_dependencies.yml

@@ -1,38 +0,0 @@
-name: Upgrade Dependencies
-on:
-    workflow_dispatch: {}
-    workflow_call:
-        secrets:
-            ELEMENT_BOT_TOKEN:
-                required: true
-jobs:
-    upgrade:
-        runs-on: ubuntu-latest
-        steps:
-            - uses: actions/checkout@v3
-
-            - uses: actions/setup-node@v3
-              with:
-                  cache: "yarn"
-
-            - name: Upgrade
-              run: yarn upgrade && yarn install
-
-            - name: Create Pull Request
-              id: cpr
-              uses: peter-evans/create-pull-request@284f54f989303d2699d373481a0cfa13ad5a6666 # v5
-              with:
-                  token: ${{ secrets.ELEMENT_BOT_TOKEN }}
-                  branch: actions/upgrade-deps
-                  delete-branch: true
-                  title: Upgrade dependencies
-                  labels: |
-                      Dependencies
-                      T-Task
-
-            - name: Enable automerge
-              run: gh pr merge --merge --auto "$PR_NUMBER"
-              if: steps.cpr.outputs.pull-request-operation == 'created'
-              env:
-                  GH_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
-                  PR_NUMBER: ${{ steps.cpr.outputs.pull-request-number }}

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname "$0")/_/husky.sh"
+
+npx lint-staged

.lintstagedrc

@@ -0,0 +1,4 @@
+{
+    "*.(ts|tsx)": ["eslint --fix", "prettier --write"],
+    "*.(py|md|yaml)": ["prettier --write"]
+}

CHANGELOG.md

@@ -1,3 +1,261 @@
+Changes in [30.3.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.3.0) (2023-12-19)
+==================================================================================================
+## ✨ Features
+
+* Element-R: disable sending room key requests ([#3939](https://github.com/matrix-org/matrix-js-sdk/pull/3939)). Contributed by @richvdh.
+
+## 🐛 Bug Fixes
+
+* Fix notifications appearing for old events ([#3946](https://github.com/matrix-org/matrix-js-sdk/pull/3946)). Contributed by @dbkr.
+* Don't back up keys that we got from backup ([#3934](https://github.com/matrix-org/matrix-js-sdk/pull/3934)). Contributed by @uhoreg.
+* Fix upload with empty Content-Type ([#3918](https://github.com/matrix-org/matrix-js-sdk/pull/3918)). Contributed by @JakubOnderka.
+* Prevent phantom notifications from events not in a room's timeline ([#3942](https://github.com/matrix-org/matrix-js-sdk/pull/3942)). Contributed by @dbkr.
+
+
+Changes in [30.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.2.0) (2023-12-05)
+==================================================================================================
+## ✨ Features
+
+* Only await key query after lazy members resolved ([#3902](https://github.com/matrix-org/matrix-js-sdk/pull/3902)). Contributed by @BillCarsonFr.
+
+## 🐛 Bug Fixes
+
+* Rewrite receipt-handling code ([#3901](https://github.com/matrix-org/matrix-js-sdk/pull/3901)). Contributed by @andybalaam.
+* Explicitly free some Rust-side objects ([#3911](https://github.com/matrix-org/matrix-js-sdk/pull/3911)). Contributed by @richvdh.
+* Fix type for TimestampToEventResponse.origin\_server\_ts ([#3906](https://github.com/matrix-org/matrix-js-sdk/pull/3906)). Contributed by @Half-Shot.
+
+
+Changes in [30.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.1.0) (2023-11-21)
+==================================================================================================
+## ✨ Features
+
+* Rotate per-participant keys when a member leaves ([#3833](https://github.com/matrix-org/matrix-js-sdk/pull/3833)). Contributed by @dbkr.
+* Add E2EE for embedded mode of Element Call ([#3667](https://github.com/matrix-org/matrix-js-sdk/pull/3667)). Contributed by @SimonBrandner.
+
+## 🐛 Bug Fixes
+
+* Shorten TimelineWindow when an event is removed ([#3862](https://github.com/matrix-org/matrix-js-sdk/pull/3862)). Contributed by @andybalaam.
+* Ignore receipts pointing at missing or invalid events ([#3817](https://github.com/matrix-org/matrix-js-sdk/pull/3817)). Contributed by @andybalaam.
+* Fix members being loaded from server on initial sync (defeating lazy loading) ([#3830](https://github.com/matrix-org/matrix-js-sdk/pull/3830)). Contributed by @BillCarsonFr.
+
+
+Changes in [30.0.1](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.0.1) (2023-11-13)
+==================================================================================================
+
+## 🐛 Bug Fixes
+ * Ensure `setUserCreator` is called when a store is assigned ([\#3867](https://github.com/matrix-org/matrix-js-sdk/pull/3867)). Fixes vector-im/element-web#26520. Contributed by @MidhunSureshR.
+
+Changes in [30.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v30.0.0) (2023-11-07)
+==================================================================================================
+
+## 🚨 BREAKING CHANGES
+ * Refactor & make base64 functions browser-safe ([\#3818](https://github.com/matrix-org/matrix-js-sdk/pull/3818)).
+
+## 🦖 Deprecations
+ * Deprecate `MatrixEvent.toJSON` ([\#3801](https://github.com/matrix-org/matrix-js-sdk/pull/3801)).
+
+## ✨ Features
+ * Element-R: Add the git sha of the binding crate to `CryptoApi#getVersion` ([\#3838](https://github.com/matrix-org/matrix-js-sdk/pull/3838)). Contributed by @florianduros.
+ * Element-R: Wire up `globalBlacklistUnverifiedDevices` field to rust crypto encryption settings ([\#3790](https://github.com/matrix-org/matrix-js-sdk/pull/3790)). Fixes vector-im/element-web#26315. Contributed by @florianduros.
+ * Element-R: Wire up room rotation ([\#3807](https://github.com/matrix-org/matrix-js-sdk/pull/3807)). Fixes vector-im/element-web#26318. Contributed by @florianduros.
+ * Element-R: Add current version of the rust-sdk and vodozemac ([\#3825](https://github.com/matrix-org/matrix-js-sdk/pull/3825)). Contributed by @florianduros.
+ * Element-R: Wire up room history visibility ([\#3805](https://github.com/matrix-org/matrix-js-sdk/pull/3805)). Fixes vector-im/element-web#26319. Contributed by @florianduros.
+ * Element-R: log when we send to-device messages ([\#3810](https://github.com/matrix-org/matrix-js-sdk/pull/3810)).
+
+## 🐛 Bug Fixes
+ * Fix reemitter not being correctly wired on user objects created in storage classes ([\#3796](https://github.com/matrix-org/matrix-js-sdk/pull/3796)). Contributed by @MidhunSureshR.
+ * Element-R: silence log errors when viewing a pending event ([\#3824](https://github.com/matrix-org/matrix-js-sdk/pull/3824)).
+ * Don't emit a closed event if the indexeddb is closed by Element ([\#3832](https://github.com/matrix-org/matrix-js-sdk/pull/3832)). Fixes vector-im/element-web#25941. Contributed by @dhenneke.
+ * Element-R: silence log errors when viewing a decryption failure ([\#3821](https://github.com/matrix-org/matrix-js-sdk/pull/3821)).
+
+Changes in [29.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v29.1.0) (2023-10-24)
+==================================================================================================
+
+## ✨ Features
+ * OIDC: refresh tokens ([\#3764](https://github.com/matrix-org/matrix-js-sdk/pull/3764)). Contributed by @kerryarchibald.
+ * OIDC: add `prompt` param to auth url creation ([\#3794](https://github.com/matrix-org/matrix-js-sdk/pull/3794)). Contributed by @kerryarchibald.
+ * Allow applications to specify their own logger instance ([\#3792](https://github.com/matrix-org/matrix-js-sdk/pull/3792)). Fixes #1899.
+ * Export AutoDiscoveryError and fix type of ALL_ERRORS ([\#3768](https://github.com/matrix-org/matrix-js-sdk/pull/3768)).
+
+## 🐛 Bug Fixes
+ * Fix sending call member events on leave ([\#3799](https://github.com/matrix-org/matrix-js-sdk/pull/3799)). Fixes vector-im/element-call#1763.
+ * Don't use event.sender in CallMembership ([\#3793](https://github.com/matrix-org/matrix-js-sdk/pull/3793)).
+ * Element-R: Don't mark QR code verification as done until it's done ([\#3791](https://github.com/matrix-org/matrix-js-sdk/pull/3791)). Fixes vector-im/element-web#26293.
+ * Element-R: Connect device to key backup when crypto is created ([\#3784](https://github.com/matrix-org/matrix-js-sdk/pull/3784)). Fixes vector-im/element-web#26316. Contributed by @florianduros.
+ * Element-R: Avoid errors in `VerificationRequest.generateQRCode` when QR code is unavailable ([\#3779](https://github.com/matrix-org/matrix-js-sdk/pull/3779)). Fixes vector-im/element-web#26300. Contributed by @florianduros.
+ * ElementR: Check key backup when user identity changes ([\#3760](https://github.com/matrix-org/matrix-js-sdk/pull/3760)). Fixes vector-im/element-web#26244. Contributed by @florianduros.
+ * Element-R: emit `VerificationRequestReceived` on incoming request ([\#3762](https://github.com/matrix-org/matrix-js-sdk/pull/3762)). Fixes vector-im/element-web#26245.
+
+Changes in [29.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v29.0.0) (2023-10-10)
+==================================================================================================
+
+## 🚨 BREAKING CHANGES
+ * Remove browserify builds ([\#3759](https://github.com/matrix-org/matrix-js-sdk/pull/3759)).
+
+## ✨ Features
+ * Export AutoDiscoveryError and fix type of ALL_ERRORS ([\#3768](https://github.com/matrix-org/matrix-js-sdk/pull/3768)).
+ * Support for stable MSC3882 get_login_token ([\#3416](https://github.com/matrix-org/matrix-js-sdk/pull/3416)). Contributed by @hughns.
+ * Remove IsUserMention and IsRoomMention from DEFAULT_OVERRIDE_RULES ([\#3752](https://github.com/matrix-org/matrix-js-sdk/pull/3752)). Contributed by @kerryarchibald.
+
+## 🐛 Bug Fixes
+ * Fix a case where joinRoom creates a duplicate Room object ([\#3747](https://github.com/matrix-org/matrix-js-sdk/pull/3747)).
+ * Add membershipID to call memberships ([\#3745](https://github.com/matrix-org/matrix-js-sdk/pull/3745)).
+ * Fix the warning for messages from unsigned devices ([\#3743](https://github.com/matrix-org/matrix-js-sdk/pull/3743)).
+ * Stop keep alive, when sync was stoped ([\#3720](https://github.com/matrix-org/matrix-js-sdk/pull/3720)). Contributed by @finsterwalder.
+
+Changes in [28.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v28.2.0) (2023-09-26)
+==================================================================================================
+
+## 🦖 Deprecations
+ * Implement `getEncryptionInfoForEvent` and deprecate `getEventEncryptionInfo` ([\#3693](https://github.com/matrix-org/matrix-js-sdk/pull/3693)).
+ * **The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. (#3189)**
+
+## ✨ Features
+ * Delete knocked room when knock membership changes ([\#3729](https://github.com/matrix-org/matrix-js-sdk/pull/3729)). Contributed by @maheichyk.
+ * Introduce MatrixRTCSession lower level group call primitive ([\#3663](https://github.com/matrix-org/matrix-js-sdk/pull/3663)).
+ * Sync knock rooms ([\#3703](https://github.com/matrix-org/matrix-js-sdk/pull/3703)). Contributed by @maheichyk.
+
+## 🐛 Bug Fixes
+ * Dont access indexed db when undefined ([\#3707](https://github.com/matrix-org/matrix-js-sdk/pull/3707)). Contributed by @finsterwalder.
+ * Don't reset unread count when adding a synthetic receipt ([\#3706](https://github.com/matrix-org/matrix-js-sdk/pull/3706)). Fixes #3684. Contributed by @andybalaam.
+
+Changes in [28.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v28.1.0) (2023-09-12)
+============================================================================================================
+
+## 🦖 Deprecations
+ * Deprecate `MatrixClient.checkUserTrust` ([\#3691](https://github.com/matrix-org/matrix-js-sdk/pull/3691)).
+ * Deprecate `MatrixClient.{prepare,create}KeyBackupVersion` in favour of new `CryptoApi.resetKeyBackup` API ([\#3689](https://github.com/matrix-org/matrix-js-sdk/pull/3689)).
+ * **The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. (#3189)**
+
+## ✨ Features
+ * Allow calls without ICE/TURN/STUN servers ([\#3695](https://github.com/matrix-org/matrix-js-sdk/pull/3695)).
+ * Emit summary update event ([\#3687](https://github.com/matrix-org/matrix-js-sdk/pull/3687)). Fixes vector-im/element-web#26033.
+ * ElementR: Update `CryptoApi.userHasCrossSigningKeys` ([\#3646](https://github.com/matrix-org/matrix-js-sdk/pull/3646)). Contributed by @florianduros.
+ * Add `join_rule` field to /publicRooms response ([\#3673](https://github.com/matrix-org/matrix-js-sdk/pull/3673)). Contributed by @charlynguyen.
+ * Use sender instead of content.creator field on m.room.create events ([\#3675](https://github.com/matrix-org/matrix-js-sdk/pull/3675)).
+
+## 🐛 Bug Fixes
+ * Provide better error for ICE Server SyntaxError ([\#3694](https://github.com/matrix-org/matrix-js-sdk/pull/3694)). Fixes vector-im/element-web#21804.
+ * Legacy crypto: re-check key backup after `bootstrapSecretStorage` ([\#3692](https://github.com/matrix-org/matrix-js-sdk/pull/3692)). Fixes vector-im/element-web#26115.
+
+Changes in [28.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v28.0.0) (2023-08-29)
+==================================================================================================
+
+## 🚨 BREAKING CHANGES
+ * Set minimum supported Matrix 1.1 version (drop legacy r0 versions) ([\#3007](https://github.com/matrix-org/matrix-js-sdk/pull/3007)). Fixes vector-im/element-web#16876.
+
+## 🦖 Deprecations
+ * **The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. (#3189)**
+
+## ✨ Features
+ * ElementR: Add `CryptoApi.requestVerificationDM` ([\#3643](https://github.com/matrix-org/matrix-js-sdk/pull/3643)). Contributed by @florianduros.
+ * Implement `CryptoApi.checkKeyBackupAndEnable` ([\#3633](https://github.com/matrix-org/matrix-js-sdk/pull/3633)). Fixes vector-im/crypto-internal#111 and vector-im/crypto-internal#112.
+
+## 🐛 Bug Fixes
+ * ElementR: Process all verification events, not just requests ([\#3650](https://github.com/matrix-org/matrix-js-sdk/pull/3650)). Contributed by @florianduros.
+
+Changes in [27.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v27.2.0) (2023-08-15)
+==================================================================================================
+
+## 🦖 Deprecations
+ * **The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. (#3189)**
+
+## ✨ Features
+ * Allow knocking rooms ([\#3647](https://github.com/matrix-org/matrix-js-sdk/pull/3647)). Contributed by @charlynguyen.
+ * Bump pagination limit to account for threaded events ([\#3638](https://github.com/matrix-org/matrix-js-sdk/pull/3638)).
+ * ElementR: Add `CryptoApi.findVerificationRequestDMInProgress` ([\#3601](https://github.com/matrix-org/matrix-js-sdk/pull/3601)). Contributed by @florianduros.
+ * Export more into the public interface ([\#3614](https://github.com/matrix-org/matrix-js-sdk/pull/3614)).
+
+## 🐛 Bug Fixes
+ * Fix wrong handling of encrypted rooms when loading them from sync accumulator ([\#3640](https://github.com/matrix-org/matrix-js-sdk/pull/3640)). Fixes vector-im/element-web#25803.
+ * Skip processing thread roots and fetching threads list when support is disabled ([\#3642](https://github.com/matrix-org/matrix-js-sdk/pull/3642)).
+ * Ensure we don't overinflate the total notification count ([\#3634](https://github.com/matrix-org/matrix-js-sdk/pull/3634)). Fixes vector-im/element-web#25803.
+
+Changes in [27.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v27.1.0) (2023-08-01)
+==================================================================================================
+
+## 🦖 Deprecations
+ * **The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. (#3189)**
+
+## ✨ Features
+ * ElementR: Add `CryptoApi.getCrossSigningKeyId` ([\#3619](https://github.com/matrix-org/matrix-js-sdk/pull/3619)). Contributed by @florianduros.
+ * ElementR: Stub `CheckOwnCrossSigningTrust`, import cross signing keys and verify local device in `bootstrapCrossSigning` ([\#3608](https://github.com/matrix-org/matrix-js-sdk/pull/3608)). Contributed by @florianduros.
+ * Specify /preview_url requests as low priority ([\#3609](https://github.com/matrix-org/matrix-js-sdk/pull/3609)). Fixes vector-im/element-web#7292.
+ * Element-R: support for displaying QR codes during verification ([\#3588](https://github.com/matrix-org/matrix-js-sdk/pull/3588)). Fixes vector-im/crypto-internal#124.
+ * Add support for scanning QR codes during verification, with Rust crypto ([\#3565](https://github.com/matrix-org/matrix-js-sdk/pull/3565)).
+ * Add methods to influence set_presence on /sync API calls ([\#3578](https://github.com/matrix-org/matrix-js-sdk/pull/3578)).
+
+## 🐛 Bug Fixes
+ * Fix threads ending up with chunks of their timelines missing ([\#3618](https://github.com/matrix-org/matrix-js-sdk/pull/3618)). Fixes vector-im/element-web#24466.
+ * Ensure we do not clobber a newer RR with an older unthreaded one ([\#3617](https://github.com/matrix-org/matrix-js-sdk/pull/3617)). Fixes vector-im/element-web#25806.
+ * Fix registration check your emails stage regression ([\#3616](https://github.com/matrix-org/matrix-js-sdk/pull/3616)).
+ * Fix how `Room::eventShouldLiveIn` handles replies to unknown parents ([\#3615](https://github.com/matrix-org/matrix-js-sdk/pull/3615)). Fixes vector-im/element-web#22603.
+ * Only send threaded read receipts if threads support is enabled ([\#3612](https://github.com/matrix-org/matrix-js-sdk/pull/3612)).
+ * ElementR: Fix `userId` parameter usage in `CryptoApi#getVerificationRequestsToDeviceInProgress` ([\#3611](https://github.com/matrix-org/matrix-js-sdk/pull/3611)). Contributed by @florianduros.
+ * Fix edge cases around non-thread relations to thread roots and read receipts ([\#3607](https://github.com/matrix-org/matrix-js-sdk/pull/3607)).
+ * Fix read receipt sending behaviour around thread roots ([\#3600](https://github.com/matrix-org/matrix-js-sdk/pull/3600)).
+ * Export typed event emitter key types ([\#3597](https://github.com/matrix-org/matrix-js-sdk/pull/3597)). Fixes #3506.
+ * Element-R: ensure that `userHasCrossSigningKeys` uses up-to-date data ([\#3599](https://github.com/matrix-org/matrix-js-sdk/pull/3599)). Fixes vector-im/element-web#25773.
+ * Fix sending `auth: null` due to broken types around UIA ([\#3594](https://github.com/matrix-org/matrix-js-sdk/pull/3594)).
+
+Changes in [27.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v27.0.0) (2023-07-18)
+==================================================================================================
+
+## 🚨 BREAKING CHANGES
+ * Drop support for Node 16 ([\#3533](https://github.com/matrix-org/matrix-js-sdk/pull/3533)).
+ * Improve types around login, registration, UIA and identity servers ([\#3537](https://github.com/matrix-org/matrix-js-sdk/pull/3537)).
+
+## 🦖 Deprecations
+ * **The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. (#3189)**
+ * Simplify `MatrixClient::setPowerLevel` API ([\#3570](https://github.com/matrix-org/matrix-js-sdk/pull/3570)). Fixes vector-im/element-web#13900 and #1844.
+ * Deprecate `VerificationRequest.getQRCodeBytes` and replace it with the asynchronous `generateQRCode`. ([\#3562](https://github.com/matrix-org/matrix-js-sdk/pull/3562)).
+ * Deprecate `VerificationRequest.beginKeyVerification()` in favour of `VerificationRequest.startVerification()`. ([\#3528](https://github.com/matrix-org/matrix-js-sdk/pull/3528)).
+ * Deprecate `Crypto.VerificationRequest` application event, replacing it with `Crypto.VerificationRequestReceived`. ([\#3514](https://github.com/matrix-org/matrix-js-sdk/pull/3514)).
+
+## ✨ Features
+ * Throw saner error when peeking has its room pulled out from under it ([\#3577](https://github.com/matrix-org/matrix-js-sdk/pull/3577)). Fixes vector-im/element-web#18679.
+ * OIDC: Log in ([\#3554](https://github.com/matrix-org/matrix-js-sdk/pull/3554)). Contributed by @kerryarchibald.
+ * Prevent threads code from making identical simultaneous API hits ([\#3541](https://github.com/matrix-org/matrix-js-sdk/pull/3541)). Fixes vector-im/element-web#25395.
+ * Update IUnsigned type to be extensible ([\#3547](https://github.com/matrix-org/matrix-js-sdk/pull/3547)).
+ * add stop() api to BackupManager for clean shutdown ([\#3553](https://github.com/matrix-org/matrix-js-sdk/pull/3553)).
+ * Log the message ID of any undecryptable to-device messages ([\#3543](https://github.com/matrix-org/matrix-js-sdk/pull/3543)).
+ * Ignore thread relations on state events for consistency with edits ([\#3540](https://github.com/matrix-org/matrix-js-sdk/pull/3540)).
+ * OIDC: validate id token ([\#3531](https://github.com/matrix-org/matrix-js-sdk/pull/3531)). Contributed by @kerryarchibald.
+
+## 🐛 Bug Fixes
+ * Fix read receipt sending behaviour around thread roots ([\#3600](https://github.com/matrix-org/matrix-js-sdk/pull/3600)).
+ * Fix `TypedEventEmitter::removeAllListeners(void)` not working ([\#3561](https://github.com/matrix-org/matrix-js-sdk/pull/3561)).
+ * Don't allow Olm unwedging rate-limiting to race ([\#3549](https://github.com/matrix-org/matrix-js-sdk/pull/3549)). Fixes vector-im/element-web#25716.
+ * Fix an instance of failed to decrypt error when an in flight `/keys/query` fails. ([\#3486](https://github.com/matrix-org/matrix-js-sdk/pull/3486)).
+ * Use the right anchor emoji for SAS verification ([\#3534](https://github.com/matrix-org/matrix-js-sdk/pull/3534)).
+ * fix a bug which caused the wrong emoji to be shown during SAS device verification. ([\#3523](https://github.com/matrix-org/matrix-js-sdk/pull/3523)).
+
+Changes in [26.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v26.2.0) (2023-07-04)
+==================================================================================================
+
+## 🦖 Deprecations
+ * The Browserify artifact is being deprecated, scheduled for removal in the October 10th release cycle. ([\#3189](https://github.com/matrix-org/matrix-js-sdk/issues/3189)).
+ * ElementR: Add `CryptoApi#bootstrapSecretStorage` ([\#3483](https://github.com/matrix-org/matrix-js-sdk/pull/3483)). Contributed by @florianduros.
+ * Deprecate `MatrixClient.findVerificationRequestDMInProgress`, `MatrixClient.getVerificationRequestsToDeviceInProgress`, and `MatrixClient.requestVerification`, in favour of methods in `CryptoApi`. ([\#3474](https://github.com/matrix-org/matrix-js-sdk/pull/3474)).
+ * Introduce a new `Crypto.VerificationRequest` interface, and deprecate direct access to the old `VerificationRequest` class. Also deprecate some related classes that were exported from `src/crypto/verification/request/VerificationRequest` ([\#3449](https://github.com/matrix-org/matrix-js-sdk/pull/3449)).
+
+## ✨ Features
+ * OIDC: navigate to authorization endpoint ([\#3499](https://github.com/matrix-org/matrix-js-sdk/pull/3499)). Contributed by @kerryarchibald.
+ * Support for interactive device verification in Element-R. ([\#3505](https://github.com/matrix-org/matrix-js-sdk/pull/3505)).
+ * Support for interactive device verification in Element-R. ([\#3508](https://github.com/matrix-org/matrix-js-sdk/pull/3508)).
+ * Support for interactive device verification in Element-R. ([\#3490](https://github.com/matrix-org/matrix-js-sdk/pull/3490)). Fixes vector-im/element-web#25316.
+ * Element-R: Store cross signing keys in secret storage ([\#3498](https://github.com/matrix-org/matrix-js-sdk/pull/3498)). Contributed by @florianduros.
+ * OIDC: add dynamic client registration util function ([\#3481](https://github.com/matrix-org/matrix-js-sdk/pull/3481)). Contributed by @kerryarchibald.
+ * Add getLastUnthreadedReceiptFor utility to Thread delegating to the underlying Room ([\#3493](https://github.com/matrix-org/matrix-js-sdk/pull/3493)).
+ * ElementR: Add `rust-crypto#createRecoveryKeyFromPassphrase` implementation ([\#3472](https://github.com/matrix-org/matrix-js-sdk/pull/3472)). Contributed by @florianduros.
+
+## 🐛 Bug Fixes
+ * Aggregate relations regardless of whether event fits into the timeline ([\#3496](https://github.com/matrix-org/matrix-js-sdk/pull/3496)). Fixes vector-im/element-web#25596.
+ * Fix bug where switching media caused media in subsequent calls to fail ([\#3489](https://github.com/matrix-org/matrix-js-sdk/pull/3489)).
+ * Fix: remove polls from room state on redaction ([\#3475](https://github.com/matrix-org/matrix-js-sdk/pull/3475)). Fixes vector-im/element-web#25573. Contributed by @kerryarchibald.
+ * Fix export type `GeneratedSecretStorageKey` ([\#3479](https://github.com/matrix-org/matrix-js-sdk/pull/3479)). Contributed by @florianduros.
+ * Close IDB database before deleting it to prevent spurious unexpected close errors ([\#3478](https://github.com/matrix-org/matrix-js-sdk/pull/3478)). Fixes vector-im/element-web#25597.
+
 Changes in [26.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v26.1.0) (2023-06-20)
 ==================================================================================================
 

README.md

@@ -11,6 +11,8 @@
 This is the [Matrix](https://matrix.org) Client-Server SDK for JavaScript and TypeScript. This SDK can be run in a
 browser or in Node.js.
 
+#### Minimum Matrix server version: v1.1
+
 The Matrix specification is constantly evolving - while this SDK aims for maximum backwards compatibility, it only
 guarantees that a feature will be supported for at least 4 spec releases. For example, if a feature the js-sdk supports
 is removed in v1.4 then the feature is _eligible_ for removal from the SDK when v1.8 is released. This SDK has no
@@ -21,17 +23,7 @@ endpoints from before Matrix 1.1, for example.
 
 ## In a browser
 
-Download the browser version from
-https://github.com/matrix-org/matrix-js-sdk/releases/latest and add that as a
-`<script>` to your page. There will be a global variable `matrixcs`
-attached to `window` through which you can access the SDK. See below for how to
-include libolm to enable end-to-end-encryption.
-
-The browser bundle supports recent versions of browsers. Typically this is ES2015
-or `> 0.5%, last 2 versions, Firefox ESR, not dead` if using
-[browserlists](https://github.com/browserslist/browserslist).
-
-Please check [the working browser example](examples/browser) for more information.
+### Note, the browserify build has been removed. Please use a bundler like webpack or vite instead.
 
 ## In Node.js
 
@@ -359,7 +351,7 @@ First, you need to pull in the right build tools:
 
 ## Building
 
-To build a browser version from scratch when developing::
+To build a browser version from scratch when developing:
 
 ```
  $ yarn build
@@ -371,9 +363,6 @@ To run tests (Jest):
  $ yarn test
 ```
 
-> **Note**
-> The `sync-browserify.spec.ts` requires a browser build (`yarn build`) in order to pass
-
 To run linting:
 
 ```

docs/SUMMARY.md

@@ -0,0 +1,9 @@
+# Summary
+
+-   [Introduction](../README.md)
+
+# Deep dive
+
+-   [Release Process](release.md)
+-   [Storage notes](storage-notes.md)
+-   [Unverified devices](warning-on-unverified-devices.md)

docs/release.md

@@ -0,0 +1,24 @@
+# Release Process
+
+## Hotfix and off-cycle releases
+
+1. Prepare the `staging` branch by using the backport automation and manually merging
+2. Go to [Releasing](#Releasing)
+
+## Release candidates
+
+1. Prepare the `staging` branch by running the [branch cut automation](https://github.com/vector-im/element-web/actions/workflows/release_prepare.yml)
+2. Go to [Releasing](#Releasing)
+
+## Releasing
+
+1. Open the [Releases page](https://github.com/matrix-org/matrix-js-sdk/releases) and inspect the draft release there
+2. Make any modifications to the release notes and tag/version as required
+3. Run [workflow](https://github.com/matrix-org/matrix-js-sdk/actions/workflows/release.yml) with the type set appropriately
+
+## Artifacts
+
+Releasing the Matrix JS SDK has just two artifacts:
+
+-   Package published to [npm](https://github.com/matrix-org/matrix-js-sdk)
+-   Docs published to [Github Pages](https://matrix-org.github.io/matrix-js-sdk/)

docs/warning-on-unverified-devices.md

@@ -1,31 +1,29 @@
 Random notes from Matthew on the two possible approaches for warning users about unexpected
 unverified devices popping up in their rooms....
 
-Original idea...
-================
+# Original idea...
 
 Warn when an existing user adds an unknown device to a room.
 
 Warn when a user joins the room with unverified or unknown devices.
 
 Warn when you initial sync if the room has any unverified devices in it.
- ^ this is good enough if we're doing local storage.
- OR, better:
+^ this is good enough if we're doing local storage.
+OR, better:
 Warn when you initial sync if the room has any new undefined devices since you were last there.
- => This means persisting the rooms that devices are in, across initial syncs.
+=> This means persisting the rooms that devices are in, across initial syncs.
 
-
-Updated idea...
-===============
+# Updated idea...
 
 Warn when the user tries to send a message:
-  - If the room has unverified devices which the user has not yet been told about in the context of this room
-    ...or in the context of this user?  currently all verification is per-user, not per-room.
+
+-   If the room has unverified devices which the user has not yet been told about in the context of this room
+    ...or in the context of this user? currently all verification is per-user, not per-room.
     ...this should be good enough.
 
-  - so track whether we have warned the user or not about unverified devices - blocked, unverified, verified, unverified_warned.
+-   so track whether we have warned the user or not about unverified devices - blocked, unverified, verified, unverified_warned.
     throw an error when trying to encrypt if there are pure unverified devices there
     app will have to search for the devices which are pure unverified to warn about them - have to do this from MembersList anyway?
-      - or megolm could warn which devices are causing the problems.
+    -   or megolm could warn which devices are causing the problems.
 
-Why do we wait to establish outbound sessions?  It just makes a horrible pause when we first try to send a message... but could otherwise unnecessarily consume resources?
+Why do we wait to establish outbound sessions? It just makes a horrible pause when we first try to send a message... but could otherwise unnecessarily consume resources?

examples/browser/README.md

@@ -1,10 +0,0 @@
-To try it out, **you must build the SDK first** and then host this folder:
-
-```
- $ yarn install
- $ yarn build
- $ cd examples/browser
- $ python -m http.server 8003
-```
-
-Then visit `http://localhost:8003`.

examples/browser/browserTest.js

@@ -1,9 +0,0 @@
-console.log("Loading browser sdk");
-
-var client = matrixcs.createClient({ baseUrl: "https://matrix.org" });
-client.publicRooms().then(function (data) {
-    console.log("data %s [...]", JSON.stringify(data).substring(0, 100));
-    console.log("Congratulations! The SDK is working on the browser!");
-    var result = document.getElementById("result");
-    result.innerHTML = "<p>The SDK appears to be working correctly.</p>";
-});

examples/browser/index.html

@@ -1,17 +0,0 @@
-<html lang="en">
-    <head>
-        <title>Test</title>
-        <meta charset="utf-8" />
-        <link rel="icon" href="data:," />
-        <script src="lib/matrix.js"></script>
-        <script src="browserTest.js"></script>
-    </head>
-    <body>
-        Sanity Testing (check the console) : This example is here to make sure that the SDK works inside a browser. It
-        simply does a GET /publicRooms on matrix.org
-        <br />
-        You should see a message confirming that the SDK works below:
-        <br />
-        <div id="result"></div>
-    </body>
-</html>

examples/browser/lib/matrix.js

@@ -1 +0,0 @@
-../../../dist/browser-matrix.js

examples/crypto-browser/lib/.gitignore

@@ -1,2 +0,0 @@
-olm.js
-olm.wasm

examples/crypto-browser/lib/matrix.js

@@ -1 +0,0 @@
-../../../dist/browser-matrix.js

examples/crypto-browser/olm-device-export-import.html

@@ -1,60 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-    <head>
-        <meta charset="UTF-8" />
-        <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-        <meta http-equiv="X-UA-Compatible" content="ie=edge" />
-        <title>Test Crypto in Browser</title>
-        <script src="lib/olm.js"></script>
-        <script src="lib/matrix.js"></script>
-    </head>
-    <body>
-        <h1>Testing export/import of Olm devices in the browser</h1>
-        <ul>
-            <li>Make sure you built the current version of the Matrix JS SDK (<code>yarn build</code>)</li>
-            <li>
-                copy <code>olm.js</code> and <code>olm.wasm</code> from a recent release of Olm (was tested with version
-                3.1.4) in directory <code>lib/</code>
-            </li>
-            <li>start a local Matrix homeserver (on port 8008, or change the port in the code)</li>
-            <li>Serve this HTML file (e.g. <code>python3 -m http.server</code>) and go to it through your browser</li>
-            <li>
-                in the JS console, do:
-                <pre>
-aliceMatrixClient = await newMatrixClient("alice-"+randomHex());
-await aliceMatrixClient.exportDevice();
-await aliceMatrixClient.getAccessToken();
-            </pre
-                >
-            </li>
-            <li>
-                copy the result of <code>exportDevice</code> and <code>getAccessToken</code> somewhere (<strong
-                    >not</strong
-                >
-                in a JS variable as it will be destroyed when you refresh the page)
-            </li>
-            <li><strong>refresh the page (F5)</strong> to make sure the client is destroyed</li>
-            <li>
-                Do the following, replacing <code>ALICE_ID</code>
-                with the user ID of Alice (you can find it in the exported data)
-                <pre>
-bobMatrixClient = await newMatrixClient("bob-"+randomHex());
-roomId = await bobMatrixClient.createEncryptedRoom([ALICE_ID]);
-await bobMatrixClient.sendTextMessage('Hi Alice!', roomId);
-            </pre
-                >
-            </li>
-            <li>Again, <strong>refresh the page (F5)</strong>. You may want to clear your console as well.</li>
-            <li>
-                Now do the following, using the exported data and the access token you saved previously:
-                <pre>
-aliceMatrixClient = await importMatrixClient(EXPORTED_DATA, ACCESS_TOKEN);
-            </pre
-                >
-            </li>
-            <li>You should see the message sent by Bob printed in the console.</li>
-        </ul>
-
-        <script src="olm-device-export-import.js"></script>
-    </body>
-</html>

examples/crypto-browser/olm-device-export-import.js

@@ -1,105 +0,0 @@
-if (!Olm) {
-    console.error("global.Olm does not seem to be present." + " Did you forget to add olm in the lib/ directory?");
-}
-
-const BASE_URL = "http://localhost:8008";
-const ROOM_CRYPTO_CONFIG = { algorithm: "m.megolm.v1.aes-sha2" };
-const PASSWORD = "password";
-
-// useful to create new usernames
-window.randomHex = () => Math.floor(Math.random() * 10 ** 6).toString(16);
-
-window.newMatrixClient = async function (username) {
-    const registrationClient = matrixcs.createClient(BASE_URL);
-
-    const userRegisterResult = await registrationClient.register(username, PASSWORD, null, { type: "m.login.dummy" });
-
-    const matrixClient = matrixcs.createClient({
-        baseUrl: BASE_URL,
-        userId: userRegisterResult.user_id,
-        accessToken: userRegisterResult.access_token,
-        deviceId: userRegisterResult.device_id,
-        sessionStore: new matrixcs.WebStorageSessionStore(window.localStorage),
-        cryptoStore: new matrixcs.MemoryCryptoStore(),
-    });
-
-    extendMatrixClient(matrixClient);
-
-    await matrixClient.initCrypto();
-    await matrixClient.startClient();
-    return matrixClient;
-};
-
-window.importMatrixClient = async function (exportedDevice, accessToken) {
-    const matrixClient = matrixcs.createClient({
-        baseUrl: BASE_URL,
-        deviceToImport: exportedDevice,
-        accessToken,
-        sessionStore: new matrixcs.WebStorageSessionStore(window.localStorage),
-        cryptoStore: new matrixcs.MemoryCryptoStore(),
-    });
-
-    extendMatrixClient(matrixClient);
-
-    await matrixClient.initCrypto();
-    await matrixClient.startClient();
-    return matrixClient;
-};
-
-function extendMatrixClient(matrixClient) {
-    // automatic join
-    matrixClient.on("RoomMember.membership", async (event, member) => {
-        if (member.membership === "invite" && member.userId === matrixClient.getUserId()) {
-            await matrixClient.joinRoom(member.roomId);
-            // setting up of room encryption seems to be triggered automatically
-            // but if we don't wait for it the first messages we send are unencrypted
-            await matrixClient.setRoomEncryption(member.roomId, { algorithm: "m.megolm.v1.aes-sha2" });
-        }
-    });
-
-    matrixClient.onDecryptedMessage = (message) => {
-        console.log("Got encrypted message: ", message);
-    };
-
-    matrixClient.on("Event.decrypted", (event) => {
-        if (event.getType() === "m.room.message") {
-            matrixClient.onDecryptedMessage(event.getContent().body);
-        } else {
-            console.log("decrypted an event of type", event.getType());
-            console.log(event);
-        }
-    });
-
-    matrixClient.createEncryptedRoom = async function (usersToInvite) {
-        const { room_id: roomId } = await this.createRoom({
-            visibility: "private",
-            invite: usersToInvite,
-        });
-
-        // matrixClient.setRoomEncryption() only updates local state
-        // but does not send anything to the server
-        // (see https://github.com/matrix-org/matrix-js-sdk/issues/905)
-        // so we do it ourselves with 'sendStateEvent'
-        await this.sendStateEvent(roomId, "m.room.encryption", ROOM_CRYPTO_CONFIG);
-        await this.setRoomEncryption(roomId, ROOM_CRYPTO_CONFIG);
-
-        // Marking all devices as verified
-        let room = this.getRoom(roomId);
-        let members = (await room.getEncryptionTargetMembers()).map((x) => x["userId"]);
-        let memberkeys = await this.downloadKeys(members);
-        for (const userId in memberkeys) {
-            for (const deviceId in memberkeys[userId]) {
-                await this.setDeviceVerified(userId, deviceId);
-            }
-        }
-
-        return roomId;
-    };
-
-    matrixClient.sendTextMessage = async function (message, roomId) {
-        return matrixClient.sendMessage(roomId, {
-            body: message,
-            msgtype: "m.text",
-        });
-    };
-}

package.json

@@ -1,26 +1,24 @@
 {
     "name": "matrix-js-sdk",
-    "version": "26.1.0",
+    "version": "30.3.0",
     "description": "Matrix Client-Server SDK for Javascript",
     "engines": {
-        "node": ">=16.0.0"
+        "node": ">=18.0.0"
     },
     "scripts": {
         "prepublishOnly": "yarn build",
         "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"",
-        "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build",
-        "clean": "rimraf lib dist",
-        "build": "yarn build:dev && yarn build:compile-browser && yarn build:minify-browser",
+        "clean": "rimraf lib",
+        "build": "yarn build:dev",
         "build:dev": "yarn clean && git rev-parse HEAD > git-revision.txt && yarn build:compile && yarn build:types",
         "build:types": "tsc -p tsconfig-build.json --emitDeclarationOnly",
         "build:compile": "babel -d lib --verbose --extensions \".ts,.js\" src",
-        "build:compile-browser": "mkdir dist && BROWSERIFYSWAP_ENV='no-rust-crypto' browserify -d src/browser-index.ts -p [ tsify -p ./tsconfig-build.json ] | exorcist dist/browser-matrix.js.map > dist/browser-matrix.js",
-        "build:minify-browser": "terser dist/browser-matrix.js --compress --mangle --source-map --output dist/browser-matrix.min.js",
         "gendoc": "typedoc",
-        "lint": "yarn lint:types && yarn lint:js",
+        "lint": "yarn lint:types && yarn lint:js && yarn lint:workflows",
         "lint:js": "eslint --max-warnings 0 src spec && prettier --check .",
         "lint:js-fix": "prettier --loglevel=warn --write . && eslint --fix src spec",
         "lint:types": "tsc --noEmit",
+        "lint:workflows": "find .github/workflows -type f \\( -iname '*.yaml' -o -iname '*.yml' \\) | xargs -I {} sh -c 'echo \"Linting {}\"; action-validator \"{}\"'",
         "test": "jest",
         "test:watch": "jest --watch",
         "coverage": "yarn test --coverage"
@@ -32,8 +30,8 @@
     "keywords": [
         "matrix-org"
     ],
-    "main": "./src/index.ts",
-    "browser": "./src/browser-index.ts",
+    "main": "./lib/index.js",
+    "browser": "./lib/browser-index.js",
     "matrix_src_main": "./src/index.ts",
     "matrix_src_browser": "./src/browser-index.ts",
     "matrix_lib_main": "./lib/index.js",
@@ -42,7 +40,6 @@
     "author": "matrix.org",
     "license": "Apache-2.0",
     "files": [
-        "dist",
         "lib",
         "src",
         "git-revision.txt",
@@ -55,19 +52,23 @@
     ],
     "dependencies": {
         "@babel/runtime": "^7.12.5",
-        "@matrix-org/matrix-sdk-crypto-js": "^0.1.0-alpha.10",
+        "@matrix-org/matrix-sdk-crypto-wasm": "^3.4.0",
         "another-json": "^0.2.0",
         "bs58": "^5.0.0",
         "content-type": "^1.0.4",
+        "jwt-decode": "^3.1.2",
         "loglevel": "^1.7.1",
         "matrix-events-sdk": "0.0.1",
-        "matrix-widget-api": "^1.3.1",
+        "matrix-widget-api": "^1.6.0",
+        "oidc-client-ts": "^2.2.4",
         "p-retry": "4",
         "sdp-transform": "^2.14.1",
         "unhomoglyph": "^1.0.6",
         "uuid": "9"
     },
     "devDependencies": {
+        "@action-validator/cli": "^0.5.3",
+        "@action-validator/core": "^0.5.3",
         "@babel/cli": "^7.12.10",
         "@babel/core": "^7.12.10",
         "@babel/eslint-parser": "^7.12.10",
@@ -80,8 +81,8 @@
         "@babel/preset-env": "^7.12.11",
         "@babel/preset-typescript": "^7.12.7",
         "@babel/register": "^7.12.10",
-        "@casualbot/jest-sonar-reporter": "^2.2.5",
-        "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz",
+        "@casualbot/jest-sonar-reporter": "2.2.7",
+        "@matrix-org/olm": "3.2.15",
         "@types/bs58": "^4.0.1",
         "@types/content-type": "^1.1.5",
         "@types/debug": "^4.1.7",
@@ -94,36 +95,31 @@
         "@typescript-eslint/parser": "^5.45.0",
         "allchange": "^1.0.6",
         "babel-jest": "^29.0.0",
-        "babelify": "^10.0.0",
-        "better-docs": "^2.4.0-beta.9",
-        "browserify": "^17.0.0",
-        "browserify-swap": "^0.2.2",
         "debug": "^4.3.4",
-        "docdash": "^2.0.0",
         "domexception": "^4.0.0",
-        "eslint": "8.41.0",
+        "eslint": "8.54.0",
         "eslint-config-google": "^0.14.0",
-        "eslint-config-prettier": "^8.5.0",
+        "eslint-config-prettier": "^9.0.0",
         "eslint-import-resolver-typescript": "^3.5.1",
         "eslint-plugin-import": "^2.26.0",
         "eslint-plugin-jest": "^27.1.6",
         "eslint-plugin-jsdoc": "^46.0.0",
         "eslint-plugin-matrix-org": "^1.0.0",
         "eslint-plugin-tsdoc": "^0.2.17",
-        "eslint-plugin-unicorn": "^47.0.0",
-        "exorcist": "^2.0.0",
-        "fake-indexeddb": "^4.0.0",
+        "eslint-plugin-unicorn": "^49.0.0",
+        "fake-indexeddb": "^5.0.0",
+        "fetch-mock": "9.11.0",
         "fetch-mock-jest": "^1.5.1",
+        "husky": "^8.0.3",
         "jest": "^29.0.0",
         "jest-environment-jsdom": "^29.0.0",
         "jest-localstorage-mock": "^2.4.6",
         "jest-mock": "^29.0.0",
+        "lint-staged": "^15.0.2",
         "matrix-mock-request": "^2.5.0",
         "prettier": "2.8.8",
         "rimraf": "^5.0.0",
-        "terser": "^5.5.1",
         "ts-node": "^10.9.1",
-        "tsify": "^5.0.2",
         "typedoc": "^0.24.0",
         "typedoc-plugin-coverage": "^2.1.0",
         "typedoc-plugin-mdn-links": "^3.0.3",
@@ -137,24 +133,5 @@
         "outputName": "jest-sonar-report.xml",
         "relativePaths": true
     },
-    "browserify": {
-        "transform": [
-            "browserify-swap",
-            [
-                "babelify",
-                {
-                    "sourceMaps": "inline",
-                    "presets": [
-                        "@babel/preset-env",
-                        "@babel/preset-typescript"
-                    ]
-                }
-            ]
-        ]
-    },
-    "browserify-swap": {
-        "no-rust-crypto": {
-            "src/rust-crypto/index.ts$": "./src/rust-crypto/browserify-index.ts"
-        }
-    }
+    "typings": "./lib/index.d.ts"
 }

post-release.sh

@@ -10,28 +10,6 @@ set -e
 jq --version > /dev/null || (echo "jq is required: please install it"; kill $$)
 
 if [ "$(git branch -lr | grep origin/develop -c)" -ge 1 ]; then
-    # When merging to develop, we need revert the `main` and `typings` fields if we adjusted them previously.
-    for i in main typings browser
-    do
-        # If a `lib` prefixed value is present, it means we adjusted the field
-        # earlier at publish time, so we should revert it now.
-        if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
-            # If there's a `src` prefixed value, use that, otherwise delete.
-            # This is used to delete the `typings` field and reset `main` back
-            # to the TypeScript source.
-            src_value=$(jq -r ".matrix_src_$i" package.json)
-            if [ "$src_value" != "null" ]; then
-                jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-            else
-                jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-            fi
-        fi
-    done
-
-    if [ -n "$(git ls-files --modified package.json)" ]; then
-        echo "Committing develop package.json"
-        git commit package.json -m "Resetting package fields for development"
-    fi
-
+    "$(dirname "$0")/scripts/release/post-merge-master.sh"
     git push origin develop
 fi

release.sh

@@ -175,18 +175,7 @@ echo "yarn version"
 # manually commit the result.
 yarn version --no-git-tag-version --new-version "$release"
 
-# For the published and dist versions of the package, we copy the
-# `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if
-# they exist). This small bit of gymnastics allows us to use the TypeScript
-# source directly for development without needing to build before linting or
-# testing.
-for i in main typings browser
-do
-    lib_value=$(jq -r ".matrix_lib_$i" package.json)
-    if [ "$lib_value" != "null" ]; then
-        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
-    fi
-done
+"$(dirname "$0")/scripts/release/pre-release.sh"
 
 # commit yarn.lock if it exists, is versioned, and is modified
 if [[ -f yarn.lock && $(git status --porcelain yarn.lock | grep '^ M') ]];

scripts/release/merge-release-notes.js

@@ -0,0 +1,104 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+
+async function getRelease(github, dependency) {
+    let owner;
+    let repo;
+    let tag;
+    if (dependency.includes("/") && dependency.includes("@")) {
+        owner = dependency.split("/")[0];
+        repo = dependency.split("/")[1].split("@")[0];
+        tag = dependency.split("@")[1];
+    } else {
+        const upstreamPackageJson = JSON.parse(fs.readFileSync(`./node_modules/${dependency}/package.json`, "utf8"));
+        [owner, repo] = upstreamPackageJson.repository.url.split("/").slice(-2);
+        tag = `v${upstreamPackageJson.version}`;
+    }
+
+    const response = await github.rest.repos.getReleaseByTag({
+        owner,
+        repo,
+        tag,
+    });
+    return response.data;
+}
+
+const HEADING_PREFIX = "## ";
+
+const main = async ({ github, releaseId, dependencies }) => {
+    const { GITHUB_REPOSITORY } = process.env;
+    const [owner, repo] = GITHUB_REPOSITORY.split("/");
+
+    const sections = new Map();
+    let heading = null;
+    for (const dependency of dependencies) {
+        const release = await getRelease(github, dependency);
+        for (const line of release.body.split("\n")) {
+            if (line.startsWith(HEADING_PREFIX)) {
+                heading = line.trim();
+                sections.set(heading, []);
+                continue;
+            }
+            if (heading && line) {
+                sections.get(heading).push(line.trim());
+            }
+        }
+    }
+
+    const { data: release } = await github.rest.repos.getRelease({
+        owner,
+        repo,
+        release_id: releaseId,
+    });
+
+    const headings = ["🚨 BREAKING CHANGES", "🦖 Deprecations", "✨ Features", "🐛 Bug Fixes", "🧰 Maintenance"].map(
+        (h) => HEADING_PREFIX + h,
+    );
+
+    heading = null;
+    const output = [];
+    for (const line of [...release.body.split("\n"), null]) {
+        if (line === null || line.startsWith(HEADING_PREFIX)) {
+            // If we have a heading, and it's not the first in the list of pending headings, output the section.
+            // If we're processing the last line (null) then output all remaining sections.
+            while (headings.length > 0 && (line === null || (heading && headings[0] !== heading))) {
+                const heading = headings.shift();
+                if (sections.has(heading)) {
+                    output.push(heading);
+                    output.push(...sections.get(heading));
+                }
+            }
+
+            if (heading && sections.has(heading)) {
+                const lastIsBlank = !output.at(-1)?.trim();
+                if (lastIsBlank) output.pop();
+                output.push(...sections.get(heading));
+                if (lastIsBlank) output.push("");
+            }
+            heading = line;
+        }
+        output.push(line);
+    }
+
+    return output.join("\n");
+};
+
+// This is just for testing locally
+// Needs environment variables GITHUB_TOKEN & GITHUB_REPOSITORY
+if (require.main === module) {
+    const { Octokit } = require("@octokit/rest");
+    const github = new Octokit({ auth: process.env.GITHUB_TOKEN });
+    if (process.argv.length < 4) {
+        // eslint-disable-next-line no-console
+        console.error("Usage: node merge-release-notes.js owner/repo:release_id npm-package-name ...");
+        process.exit(1);
+    }
+    const [releaseId, ...dependencies] = process.argv.slice(2);
+    main({ github, releaseId, dependencies }).then((output) => {
+        // eslint-disable-next-line no-console
+        console.log(output);
+    });
+}
+
+module.exports = main;

scripts/release/post-merge-master.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# When merging to develop, we need revert the `main` and `typings` fields if we adjusted them previously.
+for i in main typings browser
+do
+    # If a `lib` prefixed value is present, it means we adjusted the field earlier at publish time, so we should revert it now.
+    if [ "$(jq -r ".matrix_lib_$i" package.json)" != "null" ]; then
+        # If there's a `src` prefixed value, use that, otherwise delete.
+        # This is used to delete the `typings` field and reset `main` back to the TypeScript source.
+        src_value=$(jq -r ".matrix_src_$i" package.json)
+        if [ "$src_value" != "null" ]; then
+            jq ".$i = .matrix_src_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+        else
+            jq "del(.$i)" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+        fi
+    fi
+done
+
+if [ -n "$(git ls-files --modified package.json)" ]; then
+    echo "Committing develop package.json"
+    git commit package.json -m "Resetting package fields for development"
+fi

scripts/release/pre-release.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# For the published and dist versions of the package,
+# we copy the `matrix_lib_main` and `matrix_lib_typings` fields to `main` and `typings` (if they exist).
+# This small bit of gymnastics allows us to use the TypeScript source directly for development without
+# needing to build before linting or testing.
+
+for i in main typings browser
+do
+    lib_value=$(jq -r ".matrix_lib_$i" package.json)
+    if [ "$lib_value" != "null" ]; then
+        jq ".$i = .matrix_lib_$i" package.json > package.json.new && mv package.json.new package.json && yarn prettier --write package.json
+    fi
+done

spec/TestClient.ts

@@ -32,8 +32,6 @@ import { syncPromise } from "./test-utils/test-utils";
 import { createClient, IStartClientOpts } from "../src/matrix";
 import { ICreateClientOpts, IDownloadKeyResult, MatrixClient, PendingEventOrdering } from "../src/client";
 import { MockStorageApi } from "./MockStorageApi";
-import { encodeUri } from "../src/utils";
-import { IKeyBackupSession } from "../src/crypto/keybackup";
 import { IKeysUploadResponse, IUploadKeysRequest } from "../src/client";
 import { ISyncResponder } from "./test-utils/SyncResponder";
 
@@ -92,7 +90,7 @@ export class TestClient implements IE2EKeyReceiver, ISyncResponder {
         logger.log(this + ": starting");
         this.httpBackend.when("GET", "/versions").respond(200, {
             // we have tests that rely on support for lazy-loading members
-            versions: ["r0.5.0"],
+            versions: ["v1.1"],
         });
         this.httpBackend.when("GET", "/pushrules").respond(200, {});
         this.httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
@@ -214,21 +212,6 @@ export class TestClient implements IE2EKeyReceiver, ISyncResponder {
         });
     }
 
-    /**
-     * Set up expectations that the client will query key backups for a particular session
-     */
-    public expectKeyBackupQuery(roomId: string, sessionId: string, status: number, response: IKeyBackupSession) {
-        this.httpBackend
-            .when(
-                "GET",
-                encodeUri("/room_keys/keys/$roomId/$sessionId", {
-                    $roomId: roomId,
-                    $sessionId: sessionId,
-                }),
-            )
-            .respond(status, response);
-    }
-
     /**
      * get the uploaded curve25519 device key
      *

spec/browserify/setupTests.ts

@@ -1,34 +0,0 @@
-/*
-Copyright 2020 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import "../../dist/browser-matrix"; // uses browser-matrix instead of the src
-import type { default as BrowserMatrix } from "../../src/browser-index";
-
-// stub for browser-matrix browserify tests
-// @ts-ignore
-global.XMLHttpRequest = jest.fn();
-
-afterAll(() => {
-    // clean up XMLHttpRequest mock
-    // @ts-ignore
-    global.XMLHttpRequest = undefined;
-});
-
-// Akin to spec/setupTests.ts - but that won't affect the browser-matrix bundle
-global.matrixcs = {
-    ...global.matrixcs,
-    timeoutSignal: () => new AbortController().signal,
-} as typeof BrowserMatrix;

spec/browserify/sync-browserify.spec.ts

@@ -1,92 +0,0 @@
-/*
-Copyright 2020 The Matrix.org Foundation C.I.C.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import HttpBackend from "matrix-mock-request";
-
-import "./setupTests"; // uses browser-matrix instead of the src
-import type { MatrixClient } from "../../src";
-
-const USER_ID = "@user:test.server";
-const DEVICE_ID = "device_id";
-const ACCESS_TOKEN = "access_token";
-const ROOM_ID = "!room_id:server.test";
-
-describe("Browserify Test", function () {
-    let client: MatrixClient;
-    let httpBackend: HttpBackend;
-
-    beforeEach(() => {
-        httpBackend = new HttpBackend();
-        client = new global.matrixcs.MatrixClient({
-            baseUrl: "http://test.server",
-            userId: USER_ID,
-            accessToken: ACCESS_TOKEN,
-            deviceId: DEVICE_ID,
-            fetchFn: httpBackend.fetchFn as typeof global.fetch,
-        });
-
-        httpBackend.when("GET", "/versions").respond(200, {});
-        httpBackend.when("GET", "/pushrules").respond(200, {});
-        httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
-    });
-
-    afterEach(async () => {
-        client.stopClient();
-        client.http.abort();
-        httpBackend.verifyNoOutstandingRequests();
-        httpBackend.verifyNoOutstandingExpectation();
-        await httpBackend.stop();
-    });
-
-    it("Sync", async () => {
-        const event = {
-            type: "m.room.member",
-            room_id: ROOM_ID,
-            content: {
-                membership: "join",
-                name: "Displayname",
-            },
-            event_id: "$foobar",
-        };
-
-        const syncData = {
-            next_batch: "batch1",
-            rooms: {
-                join: {
-                    [ROOM_ID]: {
-                        timeline: {
-                            events: [event],
-                            limited: false,
-                        },
-                    },
-                },
-            },
-        };
-
-        httpBackend.when("GET", "/sync").respond(200, syncData);
-        httpBackend.when("GET", "/sync").respond(200, syncData);
-
-        const syncPromise = new Promise((r) => client.once(global.matrixcs.ClientEvent.Sync, r));
-        const unexpectedErrorFn = jest.fn();
-        client.once(global.matrixcs.ClientEvent.SyncUnexpectedError, unexpectedErrorFn);
-
-        client.startClient();
-
-        await httpBackend.flushAllExpected();
-        await syncPromise;
-        expect(unexpectedErrorFn).not.toHaveBeenCalled();
-    }, 20000); // additional timeout as this test can take quite a while
-});

spec/integ/crypto/cross-signing.spec.ts

@@ -18,8 +18,25 @@ import fetchMock from "fetch-mock-jest";
 import "fake-indexeddb/auto";
 import { IDBFactory } from "fake-indexeddb";
 
-import { CRYPTO_BACKENDS, InitCrypto } from "../../test-utils/test-utils";
-import { createClient, MatrixClient, IAuthDict, UIAuthCallback } from "../../../src";
+import { CRYPTO_BACKENDS, InitCrypto, syncPromise } from "../../test-utils/test-utils";
+import { AuthDict, createClient, CryptoEvent, MatrixClient } from "../../../src";
+import { mockInitialApiRequests, mockSetupCrossSigningRequests } from "../../test-utils/mockEndpoints";
+import { encryptAES } from "../../../src/crypto/aes";
+import { CryptoCallbacks, CrossSigningKey } from "../../../src/crypto-api";
+import { SECRET_STORAGE_ALGORITHM_V1_AES } from "../../../src/secret-storage";
+import { ISyncResponder, SyncResponder } from "../../test-utils/SyncResponder";
+import { E2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
+import {
+    MASTER_CROSS_SIGNING_PRIVATE_KEY_BASE64,
+    SELF_CROSS_SIGNING_PRIVATE_KEY_BASE64,
+    SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64,
+    SIGNED_CROSS_SIGNING_KEYS_DATA,
+    SIGNED_TEST_DEVICE_DATA,
+    USER_CROSS_SIGNING_PRIVATE_KEY_BASE64,
+} from "../../test-utils/test-data";
+import * as testData from "../../test-utils/test-data";
+import { E2EKeyResponder } from "../../test-utils/E2EKeyResponder";
+import { AccountDataAccumulator } from "../../test-utils/AccountDataAccumulator";
 
 afterEach(() => {
     // reset fake-indexeddb after each test, to make sure we don't leak connections
@@ -38,8 +55,32 @@ const TEST_DEVICE_ID = "xzcvb";
  * to provide the most effective integration tests possible.
  */
 describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: string, initCrypto: InitCrypto) => {
+    // newBackendOnly is the opposite to `oldBackendOnly`: it will skip the test if we are running against the legacy
+    // backend. Once we drop support for legacy crypto, it will go away.
+    const newBackendOnly = backend === "rust-sdk" ? test : test.skip;
+
     let aliceClient: MatrixClient;
 
+    /** an object which intercepts `/sync` requests from {@link #aliceClient} */
+    let syncResponder: ISyncResponder;
+
+    /** an object which intercepts `/keys/query` requests on the test homeserver */
+    let e2eKeyResponder: E2EKeyResponder;
+
+    // Encryption key used to encrypt cross signing keys
+    const encryptionKey = new Uint8Array(32);
+
+    /**
+     * Create the {@link CryptoCallbacks}
+     */
+    function createCryptoCallbacks(): CryptoCallbacks {
+        return {
+            getSecretStorageKey: (keys, name) => {
+                return Promise.resolve<[string, Uint8Array]>(["key_id", encryptionKey]);
+            },
+        };
+    }
+
     beforeEach(async () => {
         // anything that we don't have a specific matcher for silently returns a 404
         fetchMock.catch(404);
@@ -51,6 +92,18 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
             userId: TEST_USER_ID,
             accessToken: "akjgkrgjs",
             deviceId: TEST_DEVICE_ID,
+            cryptoCallbacks: createCryptoCallbacks(),
+        });
+
+        syncResponder = new SyncResponder(homeserverUrl);
+        e2eKeyResponder = new E2EKeyResponder(homeserverUrl);
+        /** an object which intercepts `/keys/upload` requests on the test homeserver */
+        new E2EKeyReceiver(homeserverUrl);
+
+        // Silence warnings from the backup manager
+        fetchMock.getOnce(new URL("/_matrix/client/v3/room_keys/version", homeserverUrl).toString(), {
+            status: 404,
+            body: { errcode: "M_NOT_FOUND" },
         });
 
         await initCrypto(aliceClient);
@@ -62,45 +115,14 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
     });
 
     /**
-     * Mock the requests needed to set up cross signing
-     *
-     * Return `{}` for `GET _matrix/client/r0/user/:userId/account_data/:type` request
-     * Return `{}` for `POST _matrix/client/v3/keys/signatures/upload` request (named `upload-sigs` for fetchMock check)
-     * Return `{}` for `POST /_matrix/client/(unstable|v3)/keys/device_signing/upload` request (named `upload-keys` for fetchMock check)
-     */
-    function mockSetupCrossSigningRequests(): void {
-        // have account_data requests return an empty object
-        fetchMock.get("express:/_matrix/client/r0/user/:userId/account_data/:type", {});
-
-        // we expect a request to upload signatures for our device ...
-        fetchMock.post({ url: "path:/_matrix/client/v3/keys/signatures/upload", name: "upload-sigs" }, {});
-
-        // ... and one to upload the cross-signing keys (with UIA)
-        fetchMock.post(
-            // legacy crypto uses /unstable/; /v3/ is correct
-            {
-                url: new RegExp("/_matrix/client/(unstable|v3)/keys/device_signing/upload"),
-                name: "upload-keys",
-            },
-            {},
-        );
-    }
-
-    /**
-     * Create cross-signing keys, publish the keys
-     * Mock and bootstrap all the required steps
+     * Create cross-signing keys and publish the keys
      *
      * @param authDict - The parameters to as the `auth` dict in the key upload request.
      * @see https://spec.matrix.org/v1.6/client-server-api/#authentication-types
      */
-    async function bootstrapCrossSigning(authDict: IAuthDict): Promise<void> {
-        const uiaCallback: UIAuthCallback<void> = async (makeRequest) => {
-            await makeRequest(authDict);
-        };
-
-        // now bootstrap cross signing, and check it resolves successfully
+    async function bootstrapCrossSigning(authDict: AuthDict): Promise<void> {
         await aliceClient.getCrypto()?.bootstrapCrossSigning({
-            authUploadDeviceSigningKeys: uiaCallback,
+            authUploadDeviceSigningKeys: (makeRequest) => makeRequest(authDict).then(() => undefined),
         });
     }
 
@@ -135,6 +157,141 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
                 `[${TEST_USER_ID}].[${TEST_DEVICE_ID}].signatures.[${TEST_USER_ID}].[${sskId}]`,
             );
         });
+
+        newBackendOnly("get cross signing keys from secret storage and import them", async () => {
+            // Return public cross signing keys
+            e2eKeyResponder.addCrossSigningData(SIGNED_CROSS_SIGNING_KEYS_DATA);
+
+            mockInitialApiRequests(aliceClient.getHomeserverUrl());
+
+            // Encrypt the private keys and return them in the /sync response as if they are in Secret Storage
+            const masterKey = await encryptAES(
+                MASTER_CROSS_SIGNING_PRIVATE_KEY_BASE64,
+                encryptionKey,
+                "m.cross_signing.master",
+            );
+            const selfSigningKey = await encryptAES(
+                SELF_CROSS_SIGNING_PRIVATE_KEY_BASE64,
+                encryptionKey,
+                "m.cross_signing.self_signing",
+            );
+            const userSigningKey = await encryptAES(
+                USER_CROSS_SIGNING_PRIVATE_KEY_BASE64,
+                encryptionKey,
+                "m.cross_signing.user_signing",
+            );
+
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: 1,
+                account_data: {
+                    events: [
+                        {
+                            type: "m.cross_signing.master",
+                            content: {
+                                encrypted: {
+                                    key_id: masterKey,
+                                },
+                            },
+                        },
+                        {
+                            type: "m.cross_signing.self_signing",
+                            content: {
+                                encrypted: {
+                                    key_id: selfSigningKey,
+                                },
+                            },
+                        },
+                        {
+                            type: "m.cross_signing.user_signing",
+                            content: {
+                                encrypted: {
+                                    key_id: userSigningKey,
+                                },
+                            },
+                        },
+                        {
+                            type: "m.secret_storage.key.key_id",
+                            content: {
+                                key: "key_id",
+                                algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
+                            },
+                        },
+                    ],
+                },
+            });
+            await aliceClient.startClient();
+            await syncPromise(aliceClient);
+
+            // we expect a request to upload signatures for our device ...
+            fetchMock.post({ url: "path:/_matrix/client/v3/keys/signatures/upload", name: "upload-sigs" }, {});
+
+            // we expect the UserTrustStatusChanged event to be fired after the cross signing keys import
+            const userTrustStatusChangedPromise = new Promise<string>((resolve) =>
+                aliceClient.on(CryptoEvent.UserTrustStatusChanged, resolve),
+            );
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // Check if the UserTrustStatusChanged event was fired
+            expect(await userTrustStatusChangedPromise).toBe(aliceClient.getUserId());
+
+            // Expect the signature to be uploaded
+            expect(fetchMock.called("upload-sigs")).toBeTruthy();
+            const [, sigsOpts] = fetchMock.lastCall("upload-sigs")!;
+            const body = JSON.parse(sigsOpts!.body as string);
+            // the device should have a signature with the public self cross signing keys.
+            expect(body).toHaveProperty(
+                `[${TEST_USER_ID}].[${TEST_DEVICE_ID}].signatures.[${TEST_USER_ID}].[ed25519:${SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64}]`,
+            );
+        });
+
+        it("can bootstrapCrossSigning twice", async () => {
+            mockSetupCrossSigningRequests();
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // a second call should do nothing except GET requests
+            fetchMock.mockClear();
+            await bootstrapCrossSigning(authDict);
+            const calls = fetchMock.calls((url, opts) => opts.method != "GET");
+            expect(calls.length).toEqual(0);
+        });
+
+        newBackendOnly("will upload existing cross-signing keys to an established secret storage", async () => {
+            // This rather obscure codepath covers the case that:
+            //   - 4S is set up and working
+            //   - our device has private cross-signing keys, but has not published them to 4S
+            //
+            // To arrange that, we call `bootstrapCrossSigning` on our main device, and then (pretend to) set up 4S from
+            // a *different* device. Then, when we call `bootstrapCrossSigning` again, it should do the honours.
+
+            mockSetupCrossSigningRequests();
+            const accountDataAccumulator = new AccountDataAccumulator();
+            accountDataAccumulator.interceptGetAccountData();
+
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+
+            // Pretend that another device has uploaded a 4S key
+            accountDataAccumulator.accountDataEvents.set("m.secret_storage.default_key", { key: "key_id" });
+            accountDataAccumulator.accountDataEvents.set("m.secret_storage.key.key_id", {
+                key: "keykeykey",
+                algorithm: SECRET_STORAGE_ALGORITHM_V1_AES,
+            });
+
+            // Prepare for the cross-signing keys
+            const p = accountDataAccumulator.interceptSetAccountData(":type(m.cross_signing..*)");
+
+            await bootstrapCrossSigning(authDict);
+            await p;
+
+            // The cross-signing keys should have been uploaded
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.master")).toBeTruthy();
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.self_signing")).toBeTruthy();
+            expect(accountDataAccumulator.accountDataEvents.has("m.cross_signing.user_signing")).toBeTruthy();
+        });
     });
 
     describe("getCrossSigningStatus()", () => {
@@ -187,4 +344,99 @@ describe.each(Object.entries(CRYPTO_BACKENDS))("cross-signing (%s)", (backend: s
             expect(isCrossSigningReady).toBeTruthy();
         });
     });
+
+    describe("getCrossSigningKeyId", () => {
+        /**
+         * Intercept /keys/device_signing/upload request and return the cross signing keys
+         * https://spec.matrix.org/v1.7/client-server-api/#post_matrixclientv3keysdevice_signingupload
+         *
+         * @returns the cross signing keys
+         */
+        function awaitCrossSigningKeysUpload() {
+            return new Promise<any>((resolve) => {
+                fetchMock.post(
+                    // legacy crypto uses /unstable/; /v3/ is correct
+                    {
+                        url: new RegExp("/_matrix/client/(unstable|v3)/keys/device_signing/upload"),
+                        name: "upload-keys",
+                    },
+                    (url, options) => {
+                        const content = JSON.parse(options.body as string);
+                        resolve(content);
+                        return {};
+                    },
+                    // Override the routes define in `mockSetupCrossSigningRequests`
+                    { overwriteRoutes: true },
+                );
+            });
+        }
+
+        it("should return the cross signing key id for each cross signing key", async () => {
+            mockSetupCrossSigningRequests();
+
+            // Intercept cross signing keys upload
+            const crossSigningKeysPromise = awaitCrossSigningKeysUpload();
+
+            // provide a UIA callback, so that the cross-signing keys are uploaded
+            const authDict = { type: "test" };
+            await bootstrapCrossSigning(authDict);
+            // Get the cross signing keys
+            const crossSigningKeys = await crossSigningKeysPromise;
+
+            const getPubKey = (crossSigningKey: any) => Object.values(crossSigningKey!.keys)[0];
+
+            const masterKeyId = await aliceClient.getCrypto()!.getCrossSigningKeyId();
+            expect(masterKeyId).toBe(getPubKey(crossSigningKeys.master_key));
+
+            const selfSigningKeyId = await aliceClient.getCrypto()!.getCrossSigningKeyId(CrossSigningKey.SelfSigning);
+            expect(selfSigningKeyId).toBe(getPubKey(crossSigningKeys.self_signing_key));
+
+            const userSigningKeyId = await aliceClient.getCrypto()!.getCrossSigningKeyId(CrossSigningKey.UserSigning);
+            expect(userSigningKeyId).toBe(getPubKey(crossSigningKeys.user_signing_key));
+        });
+    });
+
+    describe("crossSignDevice", () => {
+        beforeEach(async () => {
+            jest.useFakeTimers();
+
+            // make sure that there is another device which we can sign
+            e2eKeyResponder.addDeviceKeys(SIGNED_TEST_DEVICE_DATA);
+
+            // Complete initialsync, to get the outgoing requests going
+            mockInitialApiRequests(aliceClient.getHomeserverUrl());
+            syncResponder.sendOrQueueSyncResponse({ next_batch: 1 });
+            await aliceClient.startClient();
+            await syncPromise(aliceClient);
+
+            // Wait for legacy crypto to find the device
+            await jest.advanceTimersByTimeAsync(10);
+
+            const devices = await aliceClient.getCrypto()!.getUserDeviceInfo([aliceClient.getSafeUserId()]);
+            expect(devices.get(aliceClient.getSafeUserId())!.has(testData.TEST_DEVICE_ID)).toBeTruthy();
+        });
+
+        afterEach(async () => {
+            jest.useRealTimers();
+        });
+
+        it("fails for an unknown device", async () => {
+            await expect(aliceClient.getCrypto()!.crossSignDevice("unknown")).rejects.toThrow("Unknown device");
+        });
+
+        it("cross-signs the device", async () => {
+            mockSetupCrossSigningRequests();
+            await aliceClient.getCrypto()!.bootstrapCrossSigning({});
+
+            fetchMock.mockClear();
+            await aliceClient.getCrypto()!.crossSignDevice(testData.TEST_DEVICE_ID);
+
+            // check that a sig for the device was uploaded
+            const calls = fetchMock.calls("upload-sigs");
+            expect(calls.length).toEqual(1);
+            const body = JSON.parse(calls[0][1]!.body as string);
+            const deviceSig = body[aliceClient.getSafeUserId()][testData.TEST_DEVICE_ID];
+            expect(deviceSig).toHaveProperty("signatures");
+        });
+    });
 });

spec/integ/crypto/olm-utils.ts

@@ -0,0 +1,408 @@
+/*
+Copyright 2016-2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import Olm from "@matrix-org/olm";
+import anotherjson from "another-json";
+
+import { IContent, IDeviceKeys, IDownloadKeyResult, IEvent, Keys, MatrixClient, SigningKeys } from "../../../src";
+import { IE2EKeyReceiver } from "../../test-utils/E2EKeyReceiver";
+import { ISyncResponder } from "../../test-utils/SyncResponder";
+import { syncPromise } from "../../test-utils/test-utils";
+import { KeyBackupInfo } from "../../../src/crypto-api";
+
+/**
+ * @module
+ *
+ * A set of utilities for creating Olm accounts and sessions, and encrypting/decrypting with Olm/Megolm.
+ */
+
+/** Create an Olm Account object */
+export async function createOlmAccount(): Promise<Olm.Account> {
+    await Olm.init();
+    const testOlmAccount = new Olm.Account();
+    testOlmAccount.create();
+    return testOlmAccount;
+}
+
+/**
+ * Get the device keys for the test Olm Account
+ *
+ * @param olmAccount - Test olm account
+ * @param userId - The user ID to present the keys as belonging to
+ */
+export function getTestOlmAccountKeys(olmAccount: Olm.Account, userId: string, deviceId: string): IDeviceKeys {
+    const testE2eKeys = JSON.parse(olmAccount.identity_keys());
+    const testDeviceKeys: IDeviceKeys = {
+        algorithms: ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+        device_id: deviceId,
+        keys: {
+            [`curve25519:${deviceId}`]: testE2eKeys.curve25519,
+            [`ed25519:${deviceId}`]: testE2eKeys.ed25519,
+        },
+        user_id: userId,
+    };
+
+    const j = anotherjson.stringify(testDeviceKeys);
+    const sig = olmAccount.sign(j);
+    testDeviceKeys.signatures = { [userId]: { [`ed25519:${deviceId}`]: sig } };
+    return testDeviceKeys;
+}
+
+/**
+ * Bootstrap cross signing for the given Olm account.
+ *
+ * Will generate the cross signing keys and sign them with the master key, and returns the `IDownloadKeyResult`
+ * that can be directly fed into a test e2eKeyResponder.
+ *
+ * The cross-signing keys are randomly generated, similar to how the olm account keys are generated. There may not
+ * be any value in using static vectors, as the device keys change at every test run.
+ *
+ * If some `KeyBackupInfo` are provided, the `auth_data` of each backup info will be signed with the
+ * master key, meaning the backups will be then trusted after verification.
+ *
+ * @param olmAccount - The Olm account object to use for signing the device keys.
+ * @param userId - The user ID to associate with the device keys.
+ * @param deviceId - The device ID to associate with the device keys.
+ * @param keyBackupInfo - Optional key backup infos to sign with the master key.
+ * @returns A valid keys/query response that can be fed into a test e2eKeyResponder.
+ */
+export function bootstrapCrossSigningTestOlmAccount(
+    olmAccount: Olm.Account,
+    userId: string,
+    deviceId: string,
+    keyBackupInfo: KeyBackupInfo[] = [],
+): Partial<IDownloadKeyResult> {
+    const olmAliceMSK = new global.Olm.PkSigning();
+    const masterPrivkey = olmAliceMSK.generate_seed();
+    const masterPubkey = olmAliceMSK.init_with_seed(masterPrivkey);
+
+    const olmAliceUSK = new global.Olm.PkSigning();
+    const userPrivkey = olmAliceUSK.generate_seed();
+    const userPubkey = olmAliceUSK.init_with_seed(userPrivkey);
+
+    const olmAliceSSK = new global.Olm.PkSigning();
+    const sskPrivkey = olmAliceSSK.generate_seed();
+    const sskPubkey = olmAliceSSK.init_with_seed(sskPrivkey);
+
+    const mskInfo: Keys = {
+        user_id: userId,
+        usage: ["master"],
+        keys: {
+            ["ed25519:" + masterPubkey]: masterPubkey,
+        },
+    };
+
+    const sskInfo: Partial<SigningKeys> = {
+        user_id: userId,
+        usage: ["self_signing"],
+        keys: {
+            ["ed25519:" + sskPubkey]: sskPubkey,
+        },
+    };
+    // sign the ssk with the msk
+    const sskSig = olmAliceMSK.sign(anotherjson.stringify(sskInfo));
+    sskInfo.signatures = {
+        [userId]: {
+            ["ed25519:" + masterPubkey]: sskSig,
+        },
+    };
+
+    const uskInfo: Partial<SigningKeys> = {
+        user_id: userId,
+        usage: ["user_signing"],
+        keys: {
+            ["ed25519:" + userPubkey]: userPubkey,
+        },
+    };
+
+    // sign the usk with the msk
+    const uskSig = olmAliceMSK.sign(anotherjson.stringify(uskInfo));
+    uskInfo.signatures = {
+        [userId]: {
+            ["ed25519:" + masterPubkey]: uskSig,
+        },
+    };
+
+    // get the device keys and sign them with the ssk (the device is then cross signed)
+    const deviceKeys = getTestOlmAccountKeys(olmAccount, userId, deviceId);
+
+    const copy = Object.assign({}, deviceKeys);
+    delete copy.signatures;
+    const crossSignature = olmAliceSSK.sign(anotherjson.stringify(copy));
+
+    // add the signature
+    deviceKeys.signatures![userId]["ed25519:" + sskPubkey] = crossSignature;
+
+    // if we have some key backup info, sign them with the msk
+    keyBackupInfo.forEach((info) => {
+        const unsignedAuthData = Object.assign({}, info.auth_data);
+        delete unsignedAuthData.signatures;
+        const backupSignature = olmAliceMSK.sign(anotherjson.stringify(unsignedAuthData));
+
+        info.auth_data.signatures = {
+            [userId]: {
+                ["ed25519:" + masterPubkey]: backupSignature,
+            },
+        };
+    });
+
+    // clean the olm resources as we don't need them anymore
+    olmAliceMSK.free();
+    olmAliceSSK.free();
+    olmAliceUSK.free();
+
+    return {
+        master_keys: { [userId]: mskInfo },
+        user_signing_keys: { [userId]: uskInfo as SigningKeys },
+        self_signing_keys: { [userId]: sskInfo as SigningKeys },
+        device_keys: { [userId]: { [deviceId]: deviceKeys } },
+    };
+}
+
+/** start an Olm session with a given recipient */
+export async function createOlmSession(
+    olmAccount: Olm.Account,
+    recipientTestClient: IE2EKeyReceiver,
+): Promise<Olm.Session> {
+    const keys = await recipientTestClient.awaitOneTimeKeyUpload();
+    const otkId = Object.keys(keys)[0];
+    const otk = keys[otkId];
+
+    const session = new global.Olm.Session();
+    session.create_outbound(olmAccount, recipientTestClient.getDeviceKey(), otk.key);
+    return session;
+}
+
+// IToDeviceEvent isn't exported by src/sync-accumulator.ts
+export interface ToDeviceEvent {
+    content: IContent;
+    sender: string;
+    type: string;
+}
+
+/** encrypt an event with an existing olm session */
+export function encryptOlmEvent(opts: {
+    /** the sender's user id */
+    sender?: string;
+    /** the sender's curve25519 key */
+    senderKey: string;
+    /** the sender's ed25519 key */
+    senderSigningKey: string;
+    /** the olm session to use for encryption */
+    p2pSession: Olm.Session;
+    /** the recipient's user id */
+    recipient: string;
+    /** the recipient's curve25519 key */
+    recipientCurve25519Key: string;
+    /** the recipient's ed25519 key */
+    recipientEd25519Key: string;
+    /** the payload of the message */
+    plaincontent?: object;
+    /** the event type of the payload */
+    plaintype?: string;
+}): ToDeviceEvent {
+    expect(opts.senderKey).toBeTruthy();
+    expect(opts.p2pSession).toBeTruthy();
+    expect(opts.recipient).toBeTruthy();
+
+    const plaintext = {
+        content: opts.plaincontent || {},
+        recipient: opts.recipient,
+        recipient_keys: {
+            ed25519: opts.recipientEd25519Key,
+        },
+        keys: {
+            ed25519: opts.senderSigningKey,
+        },
+        sender: opts.sender || "@bob:xyz",
+        type: opts.plaintype || "m.test",
+    };
+
+    return {
+        content: {
+            algorithm: "m.olm.v1.curve25519-aes-sha2",
+            ciphertext: {
+                [opts.recipientCurve25519Key]: opts.p2pSession.encrypt(JSON.stringify(plaintext)),
+            },
+            sender_key: opts.senderKey,
+        },
+        sender: opts.sender || "@bob:xyz",
+        type: "m.room.encrypted",
+    };
+}
+
+// encrypt an event with megolm
+export function encryptMegolmEvent(opts: {
+    senderKey: string;
+    groupSession: Olm.OutboundGroupSession;
+    plaintext?: Partial<IEvent>;
+    room_id?: string;
+}): IEvent {
+    expect(opts.senderKey).toBeTruthy();
+    expect(opts.groupSession).toBeTruthy();
+
+    const plaintext = opts.plaintext || {};
+    if (!plaintext.content) {
+        plaintext.content = {
+            body: "42",
+            msgtype: "m.text",
+        };
+    }
+    if (!plaintext.type) {
+        plaintext.type = "m.room.message";
+    }
+    if (!plaintext.room_id) {
+        expect(opts.room_id).toBeTruthy();
+        plaintext.room_id = opts.room_id;
+    }
+    return encryptMegolmEventRawPlainText({
+        senderKey: opts.senderKey,
+        groupSession: opts.groupSession,
+        plaintext,
+    });
+}
+
+export function encryptMegolmEventRawPlainText(opts: {
+    senderKey: string;
+    groupSession: Olm.OutboundGroupSession;
+    plaintext: Partial<IEvent>;
+    origin_server_ts?: number;
+}): IEvent {
+    return {
+        event_id: "$test_megolm_event_" + Math.random(),
+        sender: opts.plaintext.sender ?? "@not_the_real_sender:example.com",
+        origin_server_ts: opts.plaintext.origin_server_ts ?? 1672944778000,
+        content: {
+            algorithm: "m.megolm.v1.aes-sha2",
+            ciphertext: opts.groupSession.encrypt(JSON.stringify(opts.plaintext)),
+            device_id: "testDevice",
+            sender_key: opts.senderKey,
+            session_id: opts.groupSession.session_id(),
+        },
+        type: "m.room.encrypted",
+        unsigned: {},
+    };
+}
+
+/** build an encrypted room_key event to share a group session, using an existing olm session */
+export function encryptGroupSessionKey(opts: {
+    /** recipient's user id */
+    recipient: string;
+    /** the recipient's curve25519 key */
+    recipientCurve25519Key: string;
+    /** the recipient's ed25519 key */
+    recipientEd25519Key: string;
+    /** sender's olm account */
+    olmAccount: Olm.Account;
+    /** sender's olm session with the recipient */
+    p2pSession: Olm.Session;
+    groupSession: Olm.OutboundGroupSession;
+    room_id?: string;
+}): ToDeviceEvent {
+    const senderKeys = JSON.parse(opts.olmAccount.identity_keys());
+    return encryptOlmEvent({
+        senderKey: senderKeys.curve25519,
+        senderSigningKey: senderKeys.ed25519,
+        recipient: opts.recipient,
+        recipientCurve25519Key: opts.recipientCurve25519Key,
+        recipientEd25519Key: opts.recipientEd25519Key,
+        p2pSession: opts.p2pSession,
+        plaincontent: {
+            algorithm: "m.megolm.v1.aes-sha2",
+            room_id: opts.room_id,
+            session_id: opts.groupSession.session_id(),
+            session_key: opts.groupSession.session_key(),
+        },
+        plaintype: "m.room_key",
+    });
+}
+
+/**
+ * Test utility to correctly encrypt a secret send event to a test device using the provided p2p session.
+ *
+ * @param opts - the options for the secret send event
+ * @returns the to-device event, ready to be returned in a sync response for the test device.
+ */
+export function encryptSecretSend(opts: {
+    /** the sender's user id */
+    sender: string;
+    /** recipient's user id */
+    recipient: string;
+    /** the recipient's curve25519 key */
+    recipientCurve25519Key: string;
+    /** the recipient's ed25519 key */
+    recipientEd25519Key: string;
+    /** sender's olm account */
+    olmAccount: Olm.Account;
+    /** sender's olm session with the recipient */
+    p2pSession: Olm.Session;
+    /** The requestId of the secret request that this secret send is replying. */
+    requestId: string;
+    /** The secret value */
+    secret: string;
+}): ToDeviceEvent {
+    const senderKeys = JSON.parse(opts.olmAccount.identity_keys());
+    return encryptOlmEvent({
+        sender: opts.sender,
+        senderKey: senderKeys.curve25519,
+        senderSigningKey: senderKeys.ed25519,
+        recipient: opts.recipient,
+        recipientCurve25519Key: opts.recipientCurve25519Key,
+        recipientEd25519Key: opts.recipientEd25519Key,
+        p2pSession: opts.p2pSession,
+        plaincontent: {
+            request_id: opts.requestId,
+            secret: opts.secret,
+        },
+        plaintype: "m.secret.send",
+    });
+}
+
+/**
+ * Establish an Olm Session with the test user
+ *
+ * Waits for the test user to upload their keys, then sends a /sync response with a to-device message which will
+ * establish an Olm session.
+ *
+ * @param testClient - the MatrixClient under test, which we expect to upload account keys, and to make a
+ *    /sync request which we will respond to.
+ * @param keyReceiver - an IE2EKeyReceiver which will intercept the /keys/upload request from the client under test
+ * @param syncResponder - an ISyncResponder which will intercept /sync requests from the client under test
+ * @param peerOlmAccount: an OlmAccount which will be used to initiate the Olm session.
+ */
+export async function establishOlmSession(
+    testClient: MatrixClient,
+    keyReceiver: IE2EKeyReceiver,
+    syncResponder: ISyncResponder,
+    peerOlmAccount: Olm.Account,
+): Promise<Olm.Session> {
+    const peerE2EKeys = JSON.parse(peerOlmAccount.identity_keys());
+    const p2pSession = await createOlmSession(peerOlmAccount, keyReceiver);
+    const olmEvent = encryptOlmEvent({
+        senderKey: peerE2EKeys.curve25519,
+        senderSigningKey: peerE2EKeys.ed25519,
+        recipient: testClient.getUserId()!,
+        recipientCurve25519Key: keyReceiver.getDeviceKey(),
+        recipientEd25519Key: keyReceiver.getSigningKey(),
+        p2pSession: p2pSession,
+    });
+    syncResponder.sendOrQueueSyncResponse({
+        next_batch: 1,
+        to_device: { events: [olmEvent] },
+    });
+    await syncPromise(testClient);
+    return p2pSession;
+}

spec/integ/crypto/rust-crypto.spec.ts

@@ -41,7 +41,7 @@ describe("MatrixClient.initRustCrypto", () => {
         await expect(() => unknownDeviceClient.initRustCrypto()).rejects.toThrow("unknown deviceId");
     });
 
-    it("should create the indexed dbs", async () => {
+    it("should create the indexed db", async () => {
         const matrixClient = createClient({
             baseUrl: "http://test.server",
             userId: "@alice:localhost",
@@ -53,7 +53,25 @@ describe("MatrixClient.initRustCrypto", () => {
 
         await matrixClient.initRustCrypto();
 
-        // should have two dbs now
+        // should have an indexed db now
+        const databaseNames = (await indexedDB.databases()).map((db) => db.name);
+        expect(databaseNames).toEqual(expect.arrayContaining(["matrix-js-sdk::matrix-sdk-crypto"]));
+    });
+
+    it("should create the meta db if given a pickleKey", async () => {
+        const matrixClient = createClient({
+            baseUrl: "http://test.server",
+            userId: "@alice:localhost",
+            deviceId: "aliceDevice",
+            pickleKey: "testKey",
+        });
+
+        // No databases.
+        expect(await indexedDB.databases()).toHaveLength(0);
+
+        await matrixClient.initRustCrypto();
+
+        // should have two indexed dbs now
         const databaseNames = (await indexedDB.databases()).map((db) => db.name);
         expect(databaseNames).toEqual(
             expect.arrayContaining(["matrix-js-sdk::matrix-sdk-crypto", "matrix-js-sdk::matrix-sdk-crypto-meta"]),
@@ -78,6 +96,7 @@ describe("MatrixClient.clearStores", () => {
             baseUrl: "http://test.server",
             userId: "@alice:localhost",
             deviceId: "aliceDevice",
+            pickleKey: "testKey",
         });
 
         await matrixClient.initRustCrypto();
@@ -87,4 +106,19 @@ describe("MatrixClient.clearStores", () => {
         await matrixClient.clearStores();
         expect(await indexedDB.databases()).toHaveLength(0);
     });
+
+    it("should not fail in environments without indexedDB", async () => {
+        // eslint-disable-next-line no-global-assign
+        indexedDB = undefined!;
+        const matrixClient = createClient({
+            baseUrl: "http://test.server",
+            userId: "@alice:localhost",
+            deviceId: "aliceDevice",
+        });
+
+        await matrixClient.stopClient();
+
+        await matrixClient.clearStores();
+        // No error thrown in clearStores
+    });
 });

spec/integ/matrix-client-event-emitter.spec.ts

@@ -92,9 +92,7 @@ describe("MatrixClient events", function () {
                                     type: "m.room.create",
                                     room: "!erufh:bar",
                                     user: "@foo:bar",
-                                    content: {
-                                        creator: "@foo:bar",
-                                    },
+                                    content: {},
                                 }),
                             ],
                         },
@@ -196,6 +194,37 @@ describe("MatrixClient events", function () {
             expect(fired).toBe(true);
         });
 
+        it("should emit User events when presence data is absent in first sync", async () => {
+            const MODIFIED_SYNC_DATA: any = structuredClone(SYNC_DATA);
+            delete MODIFIED_SYNC_DATA["presence"];
+            const MODIFIED_NEXT_SYNC_DATA: any = structuredClone(NEXT_SYNC_DATA);
+            MODIFIED_NEXT_SYNC_DATA.presence = {
+                events: [
+                    utils.mkPresence({
+                        user: "@foo:bar",
+                        name: "Foo Bar",
+                        presence: "online",
+                    }),
+                ],
+            };
+            httpBackend!.when("GET", "/sync").respond(200, MODIFIED_SYNC_DATA);
+            httpBackend!.when("GET", "/sync").respond(200, MODIFIED_NEXT_SYNC_DATA);
+            let fired = false;
+            client!.on(UserEvent.Presence, function (event, user) {
+                fired = true;
+                expect(user).toBeTruthy();
+                expect(event).toBeTruthy();
+                if (!user || !event) {
+                    return;
+                }
+                expect(event.event).toEqual(MODIFIED_NEXT_SYNC_DATA.presence.events[0]);
+                expect(user.presence).toEqual(MODIFIED_NEXT_SYNC_DATA.presence.events[0]?.content?.presence);
+            });
+            client!.startClient();
+            await httpBackend!.flushAllExpected();
+            expect(fired).toBe(true);
+        });
+
         it("should emit Room events", function () {
             httpBackend!.when("GET", "/sync").respond(200, SYNC_DATA);
             httpBackend!.when("GET", "/sync").respond(200, NEXT_SYNC_DATA);

spec/integ/matrix-client-event-timeline.spec.ts

@@ -107,9 +107,7 @@ const INITIAL_SYNC_DATA = {
                         utils.mkEvent({
                             type: "m.room.create",
                             user: userId,
-                            content: {
-                                creator: userId,
-                            },
+                            content: {},
                             event: false,
                         }),
                     ],
@@ -207,7 +205,7 @@ function startClient(httpBackend: HttpBackend, client: MatrixClient) {
     httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" });
     httpBackend.when("GET", "/sync").respond(200, INITIAL_SYNC_DATA);
 
-    client.startClient();
+    client.startClient({ threadSupport: true });
 
     // set up a promise which will resolve once the client is initialised
     const prom = new Promise<void>((resolve) => {
@@ -248,7 +246,7 @@ describe("getEventTimeline support", function () {
         return startClient(httpBackend, client).then(function () {
             const room = client.getRoom(roomId)!;
             const timelineSet = room!.getTimelineSets()[0];
-            expect(client.getEventTimeline(timelineSet, "event")).rejects.toBeTruthy();
+            return expect(client.getEventTimeline(timelineSet, "event")).rejects.toBeTruthy();
         });
     });
 
@@ -260,7 +258,18 @@ describe("getEventTimeline support", function () {
         return startClient(httpBackend, client).then(() => {
             const room = client.getRoom(roomId)!;
             const timelineSet = room!.getTimelineSets()[0];
-            expect(client.getEventTimeline(timelineSet, "event")).rejects.toBeFalsy();
+            httpBackend.when("GET", `/rooms/${encodeURIComponent(roomId)}/context/event`).respond(200, () => ({
+                event: {
+                    event_id: "event",
+                },
+                events_after: [],
+                events_before: [],
+                state: [],
+            }));
+            return Promise.all([
+                expect(client.getEventTimeline(timelineSet, "event")).resolves.toBeTruthy(),
+                httpBackend.flushAllExpected(),
+            ]);
         });
     });
 
@@ -271,7 +280,7 @@ describe("getEventTimeline support", function () {
 
         return startClient(httpBackend, client).then(function () {
             const timelineSet = new EventTimelineSet(undefined);
-            expect(client.getEventTimeline(timelineSet, "event")).rejects.toBeTruthy();
+            return expect(client.getEventTimeline(timelineSet, "event")).rejects.toBeTruthy();
         });
     });
 
@@ -598,12 +607,6 @@ describe("MatrixClient event timelines", function () {
             await client.stopClient(); // we don't need the client to be syncing at this time
             const room = client.getRoom(roomId)!;
 
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return THREAD_ROOT;
-                });
-
             httpBackend
                 .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
                 .respond(200, function () {
@@ -634,12 +637,6 @@ describe("MatrixClient event timelines", function () {
             const thread = room.createThread(THREAD_ROOT.event_id!, undefined, [], false);
             await httpBackend.flushAllExpected();
             const timelineSet = thread.timelineSet;
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return THREAD_ROOT;
-                });
-            await flushHttp(emitPromise(thread, ThreadEvent.Update));
 
             const timeline = await client.getEventTimeline(timelineSet, THREAD_REPLY.event_id!);
 
@@ -790,7 +787,18 @@ describe("MatrixClient event timelines", function () {
             return startClient(httpBackend, client).then(() => {
                 const room = client.getRoom(roomId)!;
                 const timelineSet = room.getTimelineSets()[0];
-                expect(client.getLatestTimeline(timelineSet)).rejects.toBeFalsy();
+                httpBackend.when("GET", `/rooms/${encodeURIComponent(roomId)}/context/event`).respond(200, () => ({
+                    event: {
+                        event_id: "event",
+                    },
+                    events_after: [],
+                    events_before: [],
+                    state: [],
+                }));
+                return Promise.all([
+                    expect(client.getEventTimeline(timelineSet, "event")).resolves.toBeTruthy(),
+                    httpBackend.flushAllExpected(),
+                ]);
             });
         });
 
@@ -1274,7 +1282,6 @@ describe("MatrixClient event timelines", function () {
             THREAD_ROOT.event_id,
             THREAD_REPLY.event_id,
             THREAD_REPLY2.getId(),
-            THREAD_ROOT_REACTION.getId(),
             THREAD_REPLY3.getId(),
         ]);
     });
@@ -1333,7 +1340,7 @@ describe("MatrixClient event timelines", function () {
         function respondToContext(event: Partial<IEvent> = THREAD_ROOT): ExpectedHttpRequest {
             const request = httpBackend.when(
                 "GET",
-                encodeUri("/_matrix/client/r0/rooms/$roomId/context/$eventId", {
+                encodeUri("/_matrix/client/v3/rooms/$roomId/context/$eventId", {
                     $roomId: roomId,
                     $eventId: event.event_id!,
                 }),
@@ -1351,7 +1358,7 @@ describe("MatrixClient event timelines", function () {
         function respondToEvent(event: Partial<IEvent> = THREAD_ROOT): ExpectedHttpRequest {
             const request = httpBackend.when(
                 "GET",
-                encodeUri("/_matrix/client/r0/rooms/$roomId/event/$eventId", {
+                encodeUri("/_matrix/client/v3/rooms/$roomId/event/$eventId", {
                     $roomId: roomId,
                     $eventId: event.event_id!,
                 }),
@@ -1362,7 +1369,7 @@ describe("MatrixClient event timelines", function () {
         function respondToMessagesRequest(): ExpectedHttpRequest {
             const request = httpBackend.when(
                 "GET",
-                encodeUri("/_matrix/client/r0/rooms/$roomId/messages", {
+                encodeUri("/_matrix/client/v3/rooms/$roomId/messages", {
                     $roomId: roomId,
                 }),
             );
@@ -1510,7 +1517,8 @@ describe("MatrixClient event timelines", function () {
                     },
                     event: true,
                 });
-                THREAD_REPLY2.localTimestamp += 1000;
+                // this has to come after THREAD_REPLY which hasn't been instantiated by us
+                THREAD_REPLY2.localTimestamp += 10000000;
 
                 // Test data for the first thread, with the second reply
                 const THREAD_ROOT_UPDATED = {
@@ -1570,9 +1578,6 @@ describe("MatrixClient event timelines", function () {
                 thread.initialEventsFetched = true;
                 const prom = emitPromise(room, ThreadEvent.NewReply);
                 respondToEvent(THREAD_ROOT_UPDATED);
-                respondToEvent(THREAD_ROOT_UPDATED);
-                respondToEvent(THREAD_ROOT_UPDATED);
-                respondToEvent(THREAD_ROOT_UPDATED);
                 respondToEvent(THREAD2_ROOT);
                 await room.addLiveEvents([THREAD_REPLY2]);
                 await httpBackend.flushAllExpected();
@@ -1699,13 +1704,11 @@ describe("MatrixClient event timelines", function () {
                 thread.initialEventsFetched = true;
                 const prom = emitPromise(room, ThreadEvent.Update);
                 respondToEvent(THREAD_ROOT_UPDATED);
-                respondToEvent(THREAD_ROOT_UPDATED);
-                respondToEvent(THREAD_ROOT_UPDATED);
                 respondToEvent(THREAD2_ROOT);
                 await room.addLiveEvents([THREAD_REPLY_REACTION]);
                 await httpBackend.flushAllExpected();
                 await prom;
-                expect(thread.length).toBe(2);
+                expect(thread.length).toBe(1); // reactions don't count towards the length of a thread
                 // Test thread order is unchanged
                 expect(timeline!.getEvents().map((it) => it.event.event_id)).toEqual([
                     THREAD_ROOT.event_id,
@@ -2047,71 +2050,7 @@ describe("MatrixClient event timelines", function () {
             expect(thread.initialEventsFetched).toBeTruthy();
             const timelineSet = thread.timelineSet;
 
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return THREAD_ROOT;
-                });
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return THREAD_ROOT;
-                });
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return THREAD_ROOT;
-                });
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/event/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return THREAD_ROOT;
-                });
-            httpBackend
-                .when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(THREAD_ROOT.event_id!))
-                .respond(200, function () {
-                    return {
-                        start: "start_token",
-                        events_before: [],
-                        event: THREAD_ROOT,
-                        events_after: [],
-                        end: "end_token",
-                        state: [],
-                    };
-                });
-            httpBackend
-                .when(
-                    "GET",
-                    "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
-                        encodeURIComponent(THREAD_ROOT.event_id!) +
-                        "/" +
-                        encodeURIComponent(THREAD_RELATION_TYPE.name) +
-                        buildRelationPaginationQuery({
-                            dir: Direction.Backward,
-                            from: "start_token",
-                        }),
-                )
-                .respond(200, function () {
-                    return {
-                        chunk: [],
-                    };
-                });
-            httpBackend
-                .when(
-                    "GET",
-                    "/_matrix/client/v1/rooms/!foo%3Abar/relations/" +
-                        encodeURIComponent(THREAD_ROOT.event_id!) +
-                        "/" +
-                        encodeURIComponent(THREAD_RELATION_TYPE.name) +
-                        buildRelationPaginationQuery({ dir: Direction.Forward, from: "end_token" }),
-                )
-                .respond(200, function () {
-                    return {
-                        chunk: [THREAD_REPLY],
-                    };
-                });
-
-            const timeline = await flushHttp(client.getEventTimeline(timelineSet, THREAD_ROOT.event_id!));
+            const timeline = await client.getEventTimeline(timelineSet, THREAD_ROOT.event_id!);
 
             httpBackend.when("GET", "/sync").respond(200, {
                 next_batch: "s_5_5",

spec/integ/matrix-client-opts.spec.ts

@@ -57,9 +57,7 @@ describe("MatrixClient opts", function () {
                                 type: "m.room.create",
                                 room: roomId,
                                 user: userId,
-                                content: {
-                                    creator: userId,
-                                },
+                                content: {},
                             }),
                         ],
                     },

spec/integ/matrix-client-room-timeline.spec.ts

@@ -85,9 +85,7 @@ describe("MatrixClient room timelines", function () {
                                 type: "m.room.create",
                                 room: roomId,
                                 user: userId,
-                                content: {
-                                    creator: userId,
-                                },
+                                content: {},
                             }),
                         ],
                     },

spec/integ/matrix-client-syncing-errors.spec.ts

@@ -0,0 +1,162 @@
+/*
+Copyright 2023 Holi Moli GmbH
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import "fake-indexeddb/auto";
+import fetchMock from "fetch-mock-jest";
+
+import { MatrixClient, ClientEvent, createClient, SyncState } from "../../src";
+
+const makeQueryablePromise = <T = void>(promise: Promise<T>) => {
+    let resolved = false;
+    let rejected = false;
+
+    // Observe the promise, saving the fulfillment in a closure scope.
+    const newPromise = promise.then(
+        (value) => {
+            resolved = true;
+            return value;
+        },
+        (error) => {
+            rejected = true;
+            throw error;
+        },
+    );
+    const isFulfilled = () => {
+        return resolved || rejected;
+    };
+    const isResolved = () => {
+        return resolved;
+    };
+    const isRejected = () => {
+        return rejected;
+    };
+    return { promise: newPromise, isFulfilled, isResolved, isRejected };
+};
+
+const queryablePromise = <T = void>() => {
+    let resolve!: (value: T | PromiseLike<T>) => void;
+    let reject!: (reason?: any) => void;
+
+    const promise = makeQueryablePromise<T>(
+        new Promise<T>((_resolve, _reject) => {
+            resolve = _resolve;
+            reject = _reject;
+        }),
+    );
+
+    return { resolve, reject, ...promise };
+};
+
+describe("MatrixClient syncing errors", () => {
+    const selfUserId = "@alice:localhost";
+    const selfAccessToken = "aseukfgwef";
+    const unknownTokenErrorData = {
+        status: 401,
+        body: {
+            errcode: "M_UNKNOWN_TOKEN",
+            error: "Invalid access token passed.",
+            soft_logout: false,
+        },
+    };
+    let client: MatrixClient | undefined;
+
+    beforeEach(() => {
+        client = createClient({
+            baseUrl: "http://tocal.test.server",
+            userId: selfUserId,
+            accessToken: selfAccessToken,
+            deviceId: "myDevice",
+        });
+    });
+
+    it("should retry, until errors are solved.", async () => {
+        jest.useFakeTimers();
+        fetchMock.config.overwriteRoutes = false;
+        fetchMock
+            .getOnce("end:versions", {}) // first version check without credentials needs to succeed
+            .getOnce("end:versions", 429) // second version check fails with 429 triggering another retry
+            .get("end:versions", {}) // further version checks succeed
+            .getOnce("end:pushrules/", 429) // first pushrules check fails starting retry
+            .get("end:pushrules/", {}) // further pushrules check succeed
+            .catch({}); // all other calls succeed
+
+        const syncEvents = Array.from({ length: 5 }, queryablePromise<SyncState>);
+
+        client!.on(ClientEvent.Sync, (state: SyncState, lastState: SyncState | null) => {
+            let i = 0;
+            for (; i < syncEvents.length && syncEvents[i].isFulfilled(); i++) {
+                // find index of first unfulfilled promise
+            }
+            syncEvents[i].resolve(state);
+        });
+
+        await client!.startClient();
+        expect(await syncEvents[0].promise).toBe(SyncState.Error);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive/sync
+        expect(await syncEvents[1].promise).toBe(SyncState.Error);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive/sync
+        expect(await syncEvents[2].promise).toBe(SyncState.Prepared);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive/sync
+        expect(await syncEvents[3].promise).toBe(SyncState.Syncing);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive/sync
+        expect(await syncEvents[4].promise).toBe(SyncState.Syncing);
+    });
+
+    it("should stop sync keep alive when client is stopped.", async () => {
+        jest.useFakeTimers();
+        fetchMock.config.overwriteRoutes = false;
+        fetchMock
+            .getOnce("end:versions", {}) // first version check without credentials needs to succeed
+            .get("end:versions", unknownTokenErrorData) // further version checks fails with 401
+            .get("end:pushrules/", 401) // fails with 401 without an error. This does happen in practice e.g. with Synapse
+            .post("end:logout", unknownTokenErrorData) // just to keep up a consistent scenario. Does not have a real effect for this testcase
+            .post("end:filter", 401); // just to keep up a consistent scenario. Does not have a real effect for this testcase
+
+        const firstSyncEvent = queryablePromise<SyncState>();
+        const secondSyncEvent = queryablePromise<SyncState>();
+        client!.on(ClientEvent.Sync, (state: SyncState, lastState: SyncState | null) => {
+            if (firstSyncEvent.isFulfilled()) secondSyncEvent.resolve(state);
+            firstSyncEvent.resolve(state);
+        });
+
+        await client!.startClient();
+        const logoutDone = queryablePromise();
+        client!
+            .logout(true)
+            .then(() => {
+                logoutDone.resolve();
+            })
+            .catch((e) => {
+                logoutDone.resolve();
+            });
+
+        const syntState = await firstSyncEvent.promise;
+        expect(syntState).toBe(SyncState.Error);
+        jest.runAllTimers(); // this will skip forward to trigger the keepAlive
+
+        jest.useRealTimers(); // we need real timer for the setTimout below to work
+
+        const timeoutPromise = makeQueryablePromise(new Promise<void>((res) => setTimeout(res, 1)));
+
+        await Promise.race([secondSyncEvent.promise, timeoutPromise.promise]);
+        // when syncing stopped, then the secondSyncEvent will never happen and the promise will not be resolved,
+        /// so the timeoutPromise will be resolved instead
+        expect(timeoutPromise.isFulfilled()).toBe(true);
+        expect(secondSyncEvent.isFulfilled()).toBe(false);
+
+        await logoutDone.promise; // wait for the logout to finish to prevent processing and logging after the test is done.
+    });
+});

spec/integ/matrix-client-syncing.spec.ts

@@ -38,6 +38,7 @@ import {
     Room,
     IndexedDBStore,
     RelationType,
+    EventType,
 } from "../../src";
 import { ReceiptType } from "../../src/@types/read_receipts";
 import { UNREAD_THREAD_NOTIFICATIONS } from "../../src/@types/sync";
@@ -222,9 +223,122 @@ describe("MatrixClient syncing", () => {
             expect(fires).toBe(3);
         });
 
-        it("should honour lazyLoadMembers if user is not a guest", () => {
-            client!.doesServerSupportLazyLoading = jest.fn().mockResolvedValue(true);
+        it("should emit RoomEvent.MyMembership for knock->leave->knock cycles", async () => {
+            await client!.initCrypto();
 
+            const roomId = "!cycles:example.org";
+
+            // First sync: an knock
+            const knockSyncRoomSection = {
+                knock: {
+                    [roomId]: {
+                        knock_state: {
+                            events: [
+                                {
+                                    type: "m.room.member",
+                                    state_key: selfUserId,
+                                    content: {
+                                        membership: "knock",
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                },
+            };
+            httpBackend!.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: knockSyncRoomSection,
+            });
+
+            // Second sync: a leave (reject of some kind)
+            httpBackend!.when("POST", "/leave").respond(200, {});
+            httpBackend!.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: {
+                    leave: {
+                        [roomId]: {
+                            account_data: { events: [] },
+                            ephemeral: { events: [] },
+                            state: {
+                                events: [
+                                    {
+                                        type: "m.room.member",
+                                        state_key: selfUserId,
+                                        content: {
+                                            membership: "leave",
+                                        },
+                                        prev_content: {
+                                            membership: "knock",
+                                        },
+                                        // XXX: And other fields required on an event
+                                    },
+                                ],
+                            },
+                            timeline: {
+                                limited: false,
+                                events: [
+                                    {
+                                        type: "m.room.member",
+                                        state_key: selfUserId,
+                                        content: {
+                                            membership: "leave",
+                                        },
+                                        prev_content: {
+                                            membership: "knock",
+                                        },
+                                        // XXX: And other fields required on an event
+                                    },
+                                ],
+                            },
+                        },
+                    },
+                },
+            });
+
+            // Third sync: another knock
+            httpBackend!.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: knockSyncRoomSection,
+            });
+
+            // First fire: an initial knock
+            let fires = 0;
+            client!.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
+                // Room, string, string
+                fires++;
+                expect(room.roomId).toBe(roomId);
+                expect(membership).toBe("knock");
+                expect(oldMembership).toBeFalsy();
+
+                // Second fire: a leave
+                client!.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
+                    fires++;
+                    expect(room.roomId).toBe(roomId);
+                    expect(membership).toBe("leave");
+                    expect(oldMembership).toBe("knock");
+
+                    // Third/final fire: a second knock
+                    client!.once(RoomEvent.MyMembership, (room, membership, oldMembership) => {
+                        fires++;
+                        expect(room.roomId).toBe(roomId);
+                        expect(membership).toBe("knock");
+                        expect(oldMembership).toBe("leave");
+                    });
+                });
+
+                // For maximum safety, "leave" the room after we register the handler
+                client!.leave(roomId);
+            });
+
+            // noinspection ES6MissingAwait
+            client!.startClient();
+            await httpBackend!.flushAllExpected();
+
+            expect(fires).toBe(3);
+        });
+
+        it("should honour lazyLoadMembers if user is not a guest", () => {
             httpBackend!
                 .when("GET", "/sync")
                 .check((req) => {
@@ -241,8 +355,6 @@ describe("MatrixClient syncing", () => {
         it("should not honour lazyLoadMembers if user is a guest", () => {
             httpBackend!.expectedRequests = [];
             httpBackend!.when("GET", "/versions").respond(200, {});
-            client!.doesServerSupportLazyLoading = jest.fn().mockResolvedValue(true);
-
             httpBackend!
                 .when("GET", "/sync")
                 .check((req) => {
@@ -296,6 +408,46 @@ describe("MatrixClient syncing", () => {
             expect(fires).toBe(1);
         });
 
+        it("should emit ClientEvent.Room when knocked while crypto is disabled", async () => {
+            const roomId = "!knock:example.org";
+
+            // First sync: a knock
+            const knockSyncRoomSection = {
+                knock: {
+                    [roomId]: {
+                        knock_state: {
+                            events: [
+                                {
+                                    type: "m.room.member",
+                                    state_key: selfUserId,
+                                    content: {
+                                        membership: "knock",
+                                    },
+                                },
+                            ],
+                        },
+                    },
+                },
+            };
+            httpBackend!.when("GET", "/sync").respond(200, {
+                ...syncData,
+                rooms: knockSyncRoomSection,
+            });
+
+            // First fire: an initial knock
+            let fires = 0;
+            client!.once(ClientEvent.Room, (room) => {
+                fires++;
+                expect(room.roomId).toBe(roomId);
+            });
+
+            // noinspection ES6MissingAwait
+            client!.startClient();
+            await httpBackend!.flushAllExpected();
+
+            expect(fires).toBe(1);
+        });
+
         it("should work when all network calls fail", async () => {
             httpBackend!.expectedRequests = [];
             httpBackend!.when("GET", "").fail(0, new Error("CORS or something"));
@@ -361,6 +513,7 @@ describe("MatrixClient syncing", () => {
                 join: {},
                 invite: {},
                 leave: {},
+                knock: {},
             },
         };
 
@@ -392,9 +545,7 @@ describe("MatrixClient syncing", () => {
                             type: "m.room.create",
                             room: roomOne,
                             user: selfUserId,
-                            content: {
-                                creator: selfUserId,
-                            },
+                            content: {},
                         }),
                     ],
                 },
@@ -580,9 +731,7 @@ describe("MatrixClient syncing", () => {
                                     type: "m.room.create",
                                     room: roomOne,
                                     user: selfUserId,
-                                    content: {
-                                        creator: selfUserId,
-                                    },
+                                    content: {},
                                 }),
                             ],
                         },
@@ -614,9 +763,7 @@ describe("MatrixClient syncing", () => {
                                     type: "m.room.create",
                                     room: roomTwo,
                                     user: selfUserId,
-                                    content: {
-                                        creator: selfUserId,
-                                    },
+                                    content: {},
                                 }),
                             ],
                         },
@@ -761,7 +908,6 @@ describe("MatrixClient syncing", () => {
                         room: roomOne,
                         user: otherUserId,
                         content: {
-                            creator: otherUserId,
                             room_version: "9",
                         },
                     });
@@ -847,7 +993,6 @@ describe("MatrixClient syncing", () => {
                         room: roomOne,
                         user: otherUserId,
                         content: {
-                            creator: otherUserId,
                             room_version: testMeta.roomVersion,
                         },
                     });
@@ -1375,9 +1520,7 @@ describe("MatrixClient syncing", () => {
                                     type: "m.room.create",
                                     room: roomOne,
                                     user: selfUserId,
-                                    content: {
-                                        creator: selfUserId,
-                                    },
+                                    content: {},
                                 }),
                             ],
                         } as Partial<IJoinedRoom>,
@@ -1474,9 +1617,7 @@ describe("MatrixClient syncing", () => {
                                     type: "m.room.create",
                                     room: roomOne,
                                     user: selfUserId,
-                                    content: {
-                                        creator: selfUserId,
-                                    },
+                                    content: {},
                                 }),
                             ],
                         },
@@ -1590,6 +1731,66 @@ describe("MatrixClient syncing", () => {
                 });
             });
         });
+
+        it("should apply encrypted notification logic for events within the same sync blob", async () => {
+            const roomId = "!room123:server";
+            const syncData = {
+                rooms: {
+                    join: {
+                        [roomId]: {
+                            ephemeral: {
+                                events: [],
+                            },
+                            timeline: {
+                                events: [
+                                    utils.mkEvent({
+                                        room: roomId,
+                                        event: true,
+                                        skey: "",
+                                        type: EventType.RoomEncryption,
+                                        content: {},
+                                    }),
+                                    utils.mkMessage({
+                                        room: roomId,
+                                        user: otherUserId,
+                                        msg: "hello",
+                                    }),
+                                ],
+                            },
+                            state: {
+                                events: [
+                                    utils.mkMembership({
+                                        room: roomId,
+                                        mship: "join",
+                                        user: otherUserId,
+                                    }),
+                                    utils.mkMembership({
+                                        room: roomId,
+                                        mship: "join",
+                                        user: selfUserId,
+                                    }),
+                                    utils.mkEvent({
+                                        type: "m.room.create",
+                                        room: roomId,
+                                        user: selfUserId,
+                                        content: {},
+                                    }),
+                                ],
+                            },
+                        },
+                    },
+                },
+            } as unknown as ISyncResponse;
+
+            httpBackend!.when("GET", "/sync").respond(200, syncData);
+            client!.startClient();
+
+            await Promise.all([httpBackend!.flushAllExpected(), awaitSyncEvent()]);
+
+            const room = client!.getRoom(roomId)!;
+            expect(room).toBeInstanceOf(Room);
+            expect(room.getRoomUnreadNotificationCount(NotificationCountType.Total)).toBe(0);
+        });
     });
 
     describe("of a room", () => {

spec/integ/matrix-client-unread-notifications.spec.ts

@@ -28,32 +28,70 @@ import {
     NotificationCountType,
     RelationType,
     Room,
+    fixNotificationCountOnDecryption,
 } from "../../src";
 import { TestClient } from "../TestClient";
 import { ReceiptType } from "../../src/@types/read_receipts";
 import { mkThread } from "../test-utils/thread";
 import { SyncState } from "../../src/sync";
 
+const userA = "@alice:localhost";
+const userB = "@bob:localhost";
+const selfUserId = userA;
+const selfAccessToken = "aseukfgwef";
+
+function setupTestClient(): [MatrixClient, HttpBackend] {
+    const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
+    const httpBackend = testClient.httpBackend;
+    const client = testClient.client;
+    httpBackend!.when("GET", "/versions").respond(200, {});
+    httpBackend!.when("GET", "/pushrules").respond(200, {});
+    httpBackend!.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
+    return [client, httpBackend];
+}
+
+describe("Notification count fixing", () => {
+    let client: MatrixClient | undefined;
+
+    beforeEach(() => {
+        [client] = setupTestClient();
+    });
+
+    it("doesn't increment notification count for events that can't be found in a room", async () => {
+        const roomId = "!room:localhost";
+
+        client!.startClient({ threadSupport: true });
+        const room = new Room(roomId, client!, selfUserId);
+        jest.spyOn(client!, "getRoom").mockImplementation((id) => (id === roomId ? room : null));
+
+        const event = new MatrixEvent({
+            room_id: roomId,
+            type: "m.reaction",
+            event_id: "$foo",
+            content: {
+                "m.relates_to": {
+                    rel_type: RelationType.Annotation,
+                    event_id: "$foo",
+                    key: "x",
+                },
+            },
+        });
+
+        jest.spyOn(event, "getPushActions").mockReturnValue({
+            notify: true,
+            tweaks: {},
+        });
+
+        fixNotificationCountOnDecryption(client!, event);
+
+        expect(room.getUnreadNotificationCount(NotificationCountType.Total)).toBe(0);
+    });
+});
+
 describe("MatrixClient syncing", () => {
-    const userA = "@alice:localhost";
-    const userB = "@bob:localhost";
-
-    const selfUserId = userA;
-    const selfAccessToken = "aseukfgwef";
-
     let client: MatrixClient | undefined;
     let httpBackend: HttpBackend | undefined;
 
-    const setupTestClient = (): [MatrixClient, HttpBackend] => {
-        const testClient = new TestClient(selfUserId, "DEVICE", selfAccessToken);
-        const httpBackend = testClient.httpBackend;
-        const client = testClient.client;
-        httpBackend!.when("GET", "/versions").respond(200, {});
-        httpBackend!.when("GET", "/pushrules").respond(200, {});
-        httpBackend!.when("POST", "/filter").respond(200, { filter_id: "a filter id" });
-        return [client, httpBackend];
-    };
-
     beforeEach(() => {
         [client, httpBackend] = setupTestClient();
     });
@@ -179,7 +217,6 @@ describe("MatrixClient syncing", () => {
                             events: [
                                 {
                                     content: {
-                                        creator: userB,
                                         room_version: "9",
                                     },
                                     origin_server_ts: 1,
@@ -377,6 +414,7 @@ describe("MatrixClient syncing", () => {
                 },
                 [Category.Leave]: {},
                 [Category.Invite]: {},
+                [Category.Knock]: {},
             },
         };
     }

spec/integ/sliding-sync-sdk.spec.ts

@@ -121,7 +121,7 @@ describe("SlidingSyncSdk", () => {
             await client!.initCrypto();
             syncOpts.cryptoCallbacks = syncOpts.crypto = client!.crypto;
         }
-        httpBackend!.when("GET", "/_matrix/client/r0/pushrules").respond(200, {});
+        httpBackend!.when("GET", "/_matrix/client/v3/pushrules").respond(200, {});
         sdk = new SlidingSyncSdk(mockSlidingSync, client, testOpts, syncOpts);
     };
 
@@ -188,7 +188,7 @@ describe("SlidingSyncSdk", () => {
                 [roomA]: {
                     name: "A",
                     required_state: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                         mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnStateEvent(EventType.RoomName, { name: "A" }, ""),
@@ -203,7 +203,7 @@ describe("SlidingSyncSdk", () => {
                     name: "B",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                         mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnEvent(EventType.RoomMessage, { body: "hello B" }),
@@ -215,7 +215,7 @@ describe("SlidingSyncSdk", () => {
                     name: "C",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                         mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnEvent(EventType.RoomMessage, { body: "hello C" }),
@@ -228,7 +228,7 @@ describe("SlidingSyncSdk", () => {
                     name: "D",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                         mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnEvent(EventType.RoomMessage, { body: "hello D" }),
@@ -264,7 +264,7 @@ describe("SlidingSyncSdk", () => {
                 [roomF]: {
                     name: "#foo:localhost",
                     required_state: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                         mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnStateEvent(EventType.RoomCanonicalAlias, { alias: "#foo:localhost" }, ""),
@@ -280,7 +280,7 @@ describe("SlidingSyncSdk", () => {
                     name: "G",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                         mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     ],
@@ -292,7 +292,7 @@ describe("SlidingSyncSdk", () => {
                     name: "H",
                     required_state: [],
                     timeline: [
-                        mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                        mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                         mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                         mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                         mkOwnEvent(EventType.RoomMessage, { body: "live event" }),
@@ -602,7 +602,7 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with Invite",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                     mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     mkOwnStateEvent(EventType.RoomMember, { membership: "invite" }, invitee),
@@ -718,7 +718,7 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with account data",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                     mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     mkOwnEvent(EventType.RoomMessage, { body: "hello" }),
@@ -922,7 +922,7 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with typing",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                     mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     mkOwnEvent(EventType.RoomMessage, { body: "hello" }),
@@ -963,7 +963,7 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with typing",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                     mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     mkOwnEvent(EventType.RoomMessage, { body: "hello" }),
@@ -1049,7 +1049,7 @@ describe("SlidingSyncSdk", () => {
                 name: "Room with receipts",
                 required_state: [],
                 timeline: [
-                    mkOwnStateEvent(EventType.RoomCreate, { creator: selfUserId }, ""),
+                    mkOwnStateEvent(EventType.RoomCreate, {}, ""),
                     mkOwnStateEvent(EventType.RoomMember, { membership: "join" }, selfUserId),
                     mkOwnStateEvent(EventType.RoomPowerLevels, { users: { [selfUserId]: 100 } }, ""),
                     {

spec/integ/sliding-sync.spec.ts

@@ -1161,11 +1161,6 @@ describe("SlidingSync", () => {
             httpBackend!.when("POST", syncUrl).check(pushTxn).respond(200, { pos: "f" }); // missing txn_id
             await httpBackend!.flushAllExpected();
 
-            // attach rejection handlers now else if we do it later Jest treats that as an unhandled rejection
-            // which is a fail.
-            expect(failPromise).rejects.toEqual(gotTxnIds[0]);
-            expect(failPromise2).rejects.toEqual(gotTxnIds[1]);
-
             const okPromise = slidingSync.setListRanges("a", [[0, 20]]);
             let txnId: string | undefined;
             httpBackend!
@@ -1180,8 +1175,12 @@ describe("SlidingSync", () => {
                         txn_id: txnId,
                     };
                 });
-            await httpBackend!.flushAllExpected();
-            await okPromise;
+            await Promise.all([
+                expect(failPromise).rejects.toEqual(gotTxnIds[0]),
+                expect(failPromise2).rejects.toEqual(gotTxnIds[1]),
+                httpBackend!.flushAllExpected(),
+                okPromise,
+            ]);
 
             expect(txnId).toBeDefined();
         });
@@ -1200,7 +1199,6 @@ describe("SlidingSync", () => {
 
             // attach rejection handlers now else if we do it later Jest treats that as an unhandled rejection
             // which is a fail.
-            expect(A).rejects.toEqual(gotTxnIds[0]);
 
             const C = slidingSync.setListRanges("a", [[0, 20]]);
             let pendingC = true;
@@ -1217,9 +1215,12 @@ describe("SlidingSync", () => {
                         txn_id: gotTxnIds[1],
                     };
                 });
-            await httpBackend!.flushAllExpected();
-            // A is rejected, see above
-            expect(B).resolves.toEqual(gotTxnIds[1]); // B is resolved
+            await Promise.all([
+                expect(A).rejects.toEqual(gotTxnIds[0]),
+                httpBackend!.flushAllExpected(),
+                // A is rejected, see above
+                expect(B).resolves.toEqual(gotTxnIds[1]), // B is resolved
+            ]);
             expect(pendingC).toBe(true); // C is pending still
         });
         it("should do nothing for unknown txn_ids", async () => {

spec/olm-loader.ts

@@ -20,7 +20,7 @@ import { logger } from "../src/logger";
 // try to load the olm library.
 try {
     // eslint-disable-next-line @typescript-eslint/no-var-requires
-    global.Olm = require("@matrix-org/olm");
+    globalThis.Olm = require("@matrix-org/olm");
     logger.log("loaded libolm");
 } catch (e) {
     logger.warn("unable to run crypto tests: libolm not available", e);

spec/test-utils/AccountDataAccumulator.ts

@@ -0,0 +1,108 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fetchMock from "fetch-mock-jest";
+import { MockOptionsMethodPut } from "fetch-mock";
+
+import { ISyncResponder } from "./SyncResponder";
+
+/**
+ *  An object which intercepts `account_data` get and set requests via fetch-mock.
+ */
+export class AccountDataAccumulator {
+    /**
+     * The account data events to be returned by the sync.
+     * Will be updated when fetchMock intercepts calls to PUT `/_matrix/client/v3/user/:userId/account_data/`.
+     * Will be used by `sendSyncResponseWithUpdatedAccountData`
+     */
+    public accountDataEvents: Map<String, any> = new Map();
+
+    /**
+     * Intercept requests to set a particular type of account data.
+     *
+     * Once it is set, its data is stored (for future return by `interceptGetAccountData` etc) and the resolved promise is
+     * resolved.
+     *
+     * @param accountDataType - type of account data to be intercepted
+     * @param opts - options to pass to fetchMock
+     * @returns a Promise which will resolve (with the content of the account data) once it is set.
+     */
+    public interceptSetAccountData(accountDataType: string, opts?: MockOptionsMethodPut): Promise<any> {
+        return new Promise((resolve) => {
+            // Called when the cross signing key is uploaded
+            fetchMock.put(
+                `express:/_matrix/client/v3/user/:userId/account_data/${accountDataType}`,
+                (url: string, options: RequestInit) => {
+                    const content = JSON.parse(options.body as string);
+                    const type = url.split("/").pop();
+                    // update account data for sync response
+                    this.accountDataEvents.set(type!, content);
+                    resolve(content);
+                    return {};
+                },
+                opts,
+            );
+        });
+    }
+
+    /**
+     * Intercept all requests to get account data
+     */
+    public interceptGetAccountData(): void {
+        fetchMock.get(
+            `express:/_matrix/client/v3/user/:userId/account_data/:type`,
+            (url) => {
+                const type = url.split("/").pop();
+                const existing = this.accountDataEvents.get(type!);
+                if (existing) {
+                    // return it
+                    return {
+                        status: 200,
+                        body: existing,
+                    };
+                } else {
+                    // 404
+                    return {
+                        status: 404,
+                        body: { errcode: "M_NOT_FOUND", error: "Account data not found." },
+                    };
+                }
+            },
+            { overwriteRoutes: true },
+        );
+    }
+
+    /**
+     * Send a sync response the current account data events.
+     */
+    public sendSyncResponseWithUpdatedAccountData(syncResponder: ISyncResponder): void {
+        try {
+            syncResponder.sendOrQueueSyncResponse({
+                next_batch: 1,
+                account_data: {
+                    events: Array.from(this.accountDataEvents, ([type, content]) => ({
+                        type: type,
+                        content: content,
+                    })),
+                },
+            });
+        } catch (err) {
+            // Might fail with "Cannot queue more than one /sync response" if called too often.
+            // It's ok if it fails here, the sync response is cumulative and will contain
+            // the latest account data.
+        }
+    }
+}

spec/test-utils/E2EKeyReceiver.ts

@@ -75,8 +75,6 @@ export class E2EKeyReceiver implements IE2EKeyReceiver {
             const listener = (url: string, options: RequestInit) =>
                 this.onKeyUploadRequest(resolveOneTimeKeys, options);
 
-            // catch both r0 and v3 variants
-            fetchMock.post(new URL("/_matrix/client/r0/keys/upload", homeserverUrl).toString(), listener);
             fetchMock.post(new URL("/_matrix/client/v3/keys/upload", homeserverUrl).toString(), listener);
         });
     }
@@ -145,6 +143,13 @@ export class E2EKeyReceiver implements IE2EKeyReceiver {
         return this.deviceKeys.keys[keyIds[0]];
     }
 
+    /**
+     * If the device keys have already been uploaded, return them. Else return null.
+     */
+    public getUploadedDeviceKeys(): IDeviceKeys | null {
+        return this.deviceKeys;
+    }
+
     /**
      * If one-time keys have already been uploaded, return them. Otherwise,
      * set up an expectation that the keys will be uploaded, and wait for

spec/test-utils/E2EKeyResponder.ts

@@ -0,0 +1,119 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fetchMock from "fetch-mock-jest";
+
+import { MapWithDefault } from "../../src/utils";
+import { IDownloadKeyResult } from "../../src";
+import { IDeviceKeys } from "../../src/@types/crypto";
+import { E2EKeyReceiver } from "./E2EKeyReceiver";
+
+/**
+ * An object which intercepts `/keys/query` fetches via fetch-mock.
+ */
+export class E2EKeyResponder {
+    private deviceKeysByUserByDevice = new MapWithDefault<string, Map<string, any>>(() => new Map());
+    private e2eKeyReceiversByUser = new Map<string, E2EKeyReceiver>();
+    private masterKeysByUser: Record<string, any> = {};
+    private selfSigningKeysByUser: Record<string, any> = {};
+    private userSigningKeysByUser: Record<string, any> = {};
+
+    /**
+     * Construct a new E2EKeyResponder.
+     *
+     * It will immediately register an intercept of `/keys/query` requests for the given homeserverUrl.
+     * Only /query requests made to this server will be intercepted: this allows a single test to use more than one
+     * client and have the keys collected separately.
+     *
+     * @param homeserverUrl - the Homeserver Url of the client under test.
+     */
+    public constructor(homeserverUrl: string) {
+        // set up a listener for /keys/query.
+        const listener = (url: string, options: RequestInit) => this.onKeyQueryRequest(options);
+        fetchMock.post(new URL("/_matrix/client/v3/keys/query", homeserverUrl).toString(), listener);
+    }
+
+    private onKeyQueryRequest(options: RequestInit) {
+        const content = JSON.parse(options.body as string);
+        const usersToReturn = Object.keys(content["device_keys"]);
+        const response = {
+            device_keys: {} as { [userId: string]: any },
+            master_keys: {} as { [userId: string]: any },
+            self_signing_keys: {} as { [userId: string]: any },
+            user_signing_keys: {} as { [userId: string]: any },
+            failures: {} as { [serverName: string]: any },
+        };
+        for (const user of usersToReturn) {
+            const userKeys = this.deviceKeysByUserByDevice.get(user);
+            if (userKeys !== undefined) {
+                response.device_keys[user] = Object.fromEntries(userKeys.entries());
+            }
+
+            const e2eKeyReceiver = this.e2eKeyReceiversByUser.get(user);
+            if (e2eKeyReceiver !== undefined) {
+                const deviceKeys = e2eKeyReceiver.getUploadedDeviceKeys();
+                if (deviceKeys !== null) {
+                    response.device_keys[user] ??= {};
+                    response.device_keys[user][deviceKeys.device_id] = deviceKeys;
+                }
+            }
+
+            if (this.masterKeysByUser.hasOwnProperty(user)) {
+                response.master_keys[user] = this.masterKeysByUser[user];
+            }
+            if (this.selfSigningKeysByUser.hasOwnProperty(user)) {
+                response.self_signing_keys[user] = this.selfSigningKeysByUser[user];
+            }
+            if (this.userSigningKeysByUser.hasOwnProperty(user)) {
+                response.user_signing_keys[user] = this.userSigningKeysByUser[user];
+            }
+        }
+        return response;
+    }
+
+    /**
+     * Add a set of device keys for return by a future `/keys/query`, as if they had been `/upload`ed
+     *
+     * @param keys - device keys for this device.
+     */
+    public addDeviceKeys(keys: IDeviceKeys) {
+        this.deviceKeysByUserByDevice.getOrCreate(keys.user_id).set(keys.device_id, keys);
+    }
+
+    /** Add a set of cross-signing keys for return by a future `/keys/query`, as if they had been `/keys/device_signing/upload`ed
+     *
+     * @param data cross-signing data
+     */
+    public addCrossSigningData(
+        data: Pick<IDownloadKeyResult, "master_keys" | "self_signing_keys" | "user_signing_keys">,
+    ) {
+        Object.assign(this.masterKeysByUser, data.master_keys);
+        Object.assign(this.selfSigningKeysByUser, data.self_signing_keys);
+        Object.assign(this.userSigningKeysByUser, data.user_signing_keys);
+    }
+
+    /**
+     * Add an E2EKeyReceiver to poll for uploaded keys
+     *
+     * Any keys which have been uploaded to the given `E2EKeyReceiver` at the time of the `/keys/query` request will
+     * be added to the response.
+     *
+     * @param e2eKeyReceiver
+     */
+    public addKeyReceiver(userId: string, e2eKeyReceiver: E2EKeyReceiver) {
+        this.e2eKeyReceiversByUser.set(userId, e2eKeyReceiver);
+    }
+}

spec/test-utils/SyncResponder.ts

@@ -75,7 +75,7 @@ export class SyncResponder implements ISyncResponder {
      */
     public constructor(homeserverUrl: string) {
         this.debug = debugFunc(`sync-responder:[${homeserverUrl}]`);
-        fetchMock.get("begin:" + new URL("/_matrix/client/r0/sync?", homeserverUrl).toString(), (_url, _options) =>
+        fetchMock.get("begin:" + new URL("/_matrix/client/v3/sync?", homeserverUrl).toString(), (_url, _options) =>
             this.onSyncRequest(),
         );
     }

spec/test-utils/client.ts

@@ -86,7 +86,6 @@ export const mockClientMethodsEvents = () => ({
  * Returns basic mocked client methods related to server support
  */
 export const mockClientMethodsServer = (): Partial<Record<MethodLikeKeys<MatrixClient>, unknown>> => ({
-    doesServerSupportSeparateAddAndBind: jest.fn(),
     getIdentityServerUrl: jest.fn(),
     getHomeserverUrl: jest.fn(),
     getCapabilities: jest.fn().mockReturnValue({}),

spec/test-utils/mockEndpoints.ts

@@ -16,15 +16,77 @@ limitations under the License.
 
 import fetchMock from "fetch-mock-jest";
 
+import { KeyBackupInfo } from "../../src/crypto-api";
+
 /**
  * Mock out the endpoints that the js-sdk calls when we call `MatrixClient.start()`.
  *
  * @param homeserverUrl - the homeserver url for the client under test
  */
 export function mockInitialApiRequests(homeserverUrl: string) {
-    fetchMock.getOnce(new URL("/_matrix/client/versions", homeserverUrl).toString(), { versions: ["r0.5.0"] });
-    fetchMock.getOnce(new URL("/_matrix/client/r0/pushrules/", homeserverUrl).toString(), {});
-    fetchMock.postOnce(new URL("/_matrix/client/r0/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(), {
+    fetchMock.getOnce(new URL("/_matrix/client/versions", homeserverUrl).toString(), { versions: ["v1.1"] });
+    fetchMock.getOnce(new URL("/_matrix/client/v3/pushrules/", homeserverUrl).toString(), {});
+    fetchMock.postOnce(new URL("/_matrix/client/v3/user/%40alice%3Alocalhost/filter", homeserverUrl).toString(), {
         filter_id: "fid",
     });
 }
+
+/**
+ * Mock the requests needed to set up cross signing
+ *
+ * Return 404 error for `GET _matrix/client/v3/user/:userId/account_data/:type` request
+ * Return `{}` for `POST _matrix/client/v3/keys/signatures/upload` request (named `upload-sigs` for fetchMock check)
+ * Return `{}` for `POST /_matrix/client/(unstable|v3)/keys/device_signing/upload` request (named `upload-keys` for fetchMock check)
+ */
+export function mockSetupCrossSigningRequests(): void {
+    // have account_data requests return an empty object
+    fetchMock.get("express:/_matrix/client/v3/user/:userId/account_data/:type", {
+        status: 404,
+        body: { errcode: "M_NOT_FOUND", error: "Account data not found." },
+    });
+
+    // we expect a request to upload signatures for our device ...
+    fetchMock.post({ url: "path:/_matrix/client/v3/keys/signatures/upload", name: "upload-sigs" }, {});
+
+    // ... and one to upload the cross-signing keys (with UIA)
+    fetchMock.post(
+        // legacy crypto uses /unstable/; /v3/ is correct
+        {
+            url: new RegExp("/_matrix/client/(unstable|v3)/keys/device_signing/upload"),
+            name: "upload-keys",
+        },
+        {},
+    );
+}
+
+/**
+ * Mock out requests to `/room_keys/version`.
+ *
+ * Returns `404 M_NOT_FOUND` for GET requests until `POST room_keys/version` is called.
+ * Once the POST is done, `GET /room_keys/version` will return the posted backup
+ * instead of 404.
+ *
+ * @param backupVersion - The backup version that will be returned by `POST room_keys/version`.
+ */
+export function mockSetupMegolmBackupRequests(backupVersion: string): void {
+    fetchMock.get("path:/_matrix/client/v3/room_keys/version", {
+        status: 404,
+        body: {
+            errcode: "M_NOT_FOUND",
+            error: "No current backup version",
+        },
+    });
+
+    fetchMock.post("path:/_matrix/client/v3/room_keys/version", (url, request) => {
+        const backupData: KeyBackupInfo = JSON.parse(request.body?.toString() ?? "{}");
+        backupData.version = backupVersion;
+        backupData.count = 0;
+        backupData.etag = "zer";
+        fetchMock.get("path:/_matrix/client/v3/room_keys/version", backupData, {
+            overwriteRoutes: true,
+        });
+        return {
+            version: backupVersion,
+        };
+    });
+}

spec/test-utils/oidc.ts

@@ -0,0 +1,53 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { OidcClientConfig } from "../../src";
+import { ValidatedIssuerMetadata } from "../../src/oidc/validate";
+
+/**
+ * Makes a valid OidcClientConfig with minimum valid values
+ * @param issuer used as the base for all other urls
+ * @returns OidcClientConfig
+ */
+export const makeDelegatedAuthConfig = (issuer = "https://auth.org/"): OidcClientConfig => {
+    const metadata = mockOpenIdConfiguration(issuer);
+
+    return {
+        issuer,
+        account: issuer + "account",
+        registrationEndpoint: metadata.registration_endpoint,
+        authorizationEndpoint: metadata.authorization_endpoint,
+        tokenEndpoint: metadata.token_endpoint,
+        metadata,
+    };
+};
+
+/**
+ * Useful for mocking <issuer>/.well-known/openid-configuration
+ * @param issuer used as the base for all other urls
+ * @returns ValidatedIssuerMetadata
+ */
+export const mockOpenIdConfiguration = (issuer = "https://auth.org/"): ValidatedIssuerMetadata => ({
+    issuer,
+    revocation_endpoint: issuer + "revoke",
+    token_endpoint: issuer + "token",
+    authorization_endpoint: issuer + "auth",
+    registration_endpoint: issuer + "registration",
+    jwks_uri: issuer + "jwks",
+    response_types_supported: ["code"],
+    grant_types_supported: ["authorization_code", "refresh_token"],
+    code_challenge_methods_supported: ["S256"],
+});

spec/test-utils/test-data/generate-test-data.py

@@ -26,52 +26,56 @@ python -m venv env
 
 import base64
 import json
+import base58
 
 from canonicaljson import encode_canonical_json
-from cryptography.hazmat.primitives.asymmetric import ed25519
+from cryptography.hazmat.primitives.asymmetric import ed25519, x25519
 from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat
+from cryptography.hazmat.primitives import hashes, padding, hmac
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 
-# input data
-TEST_USER_ID = "@alice:localhost"
-TEST_DEVICE_ID = "test_device"
-# any 32-byte string can be an ed25519 private key.
-TEST_DEVICE_PRIVATE_KEY_BYTES = b"deadbeefdeadbeefdeadbeefdeadbeef"
+from random import randbytes, seed
 
-MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES = b"doyouspeakwhaaaaaaaaaaaaaaaaaale"
-USER_CROSS_SIGNING_PRIVATE_KEY_BYTES = b"useruseruseruseruseruseruseruser"
-SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES = b"selfselfselfselfselfselfselfself"
+ALICE_DATA = {
+    "TEST_USER_ID": "@alice:localhost",
+    "TEST_DEVICE_ID": "test_device",
+    "TEST_ROOM_ID": "!room:id",
+    # any 32-byte string can be an ed25519 private key.
+    "TEST_DEVICE_PRIVATE_KEY_BYTES": b"deadbeefdeadbeefdeadbeefdeadbeef",
+    # any 32-byte string can be an curve25519 private key.
+    "TEST_DEVICE_CURVE_PRIVATE_KEY_BYTES": b"deadmuledeadmuledeadmuledeadmule",
 
+    "MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"doyouspeakwhaaaaaaaaaaaaaaaaaale",
+    "USER_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"useruseruseruseruseruseruseruser",
+    "SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"selfselfselfselfselfselfselfself",
+
+    # Private key for secure key backup. There are some sessions encrypted with this key in megolm-backup.spec.ts
+    "B64_BACKUP_DECRYPTION_KEY": "dwdtCnMYpX08FsFyUbJmRd9ML4frwJkqsXf7pR25LCo=",
+
+    "OTK": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw"
+}
+
+BOB_DATA = {
+    "TEST_USER_ID": "@bob:xyz",
+    "TEST_DEVICE_ID": "bob_device",
+    "TEST_ROOM_ID": "!room:id",
+    # any 32-byte string can be an ed25519 private key.
+    "TEST_DEVICE_PRIVATE_KEY_BYTES": b"Deadbeefdeadbeefdeadbeefdeadbeef",
+    # any 32-byte string can be an curve25519 private key.
+    "TEST_DEVICE_CURVE_PRIVATE_KEY_BYTES": b"Deadmuledeadmuledeadmuledeadmule",
+
+    "MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"Doyouspeakwhaaaaaaaaaaaaaaaaaale",
+    "USER_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"Useruseruseruseruseruseruseruser",
+    "SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES": b"Selfselfselfselfselfselfselfself",
+
+    # Private key for secure key backup. There are some sessions encrypted with this key in megolm-backup.spec.ts
+    "B64_BACKUP_DECRYPTION_KEY": "DwdtCnMYpX08FsFyUbJmRd9ML4frwJkqsXf7pR25LCo=",
+
+    "OTK": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw"
+}
 
 def main() -> None:
-    private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
-        TEST_DEVICE_PRIVATE_KEY_BYTES
-    )
-    b64_public_key = encode_base64(
-        private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
-    )
-
-    device_data = {
-        "algorithms": ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
-        "device_id": TEST_DEVICE_ID,
-        "keys": {
-            f"curve25519:{TEST_DEVICE_ID}": "F4uCNNlcbRvc7CfBz95ZGWBvY1ALniG1J8+6rhVoKS0",
-            f"ed25519:{TEST_DEVICE_ID}": b64_public_key,
-        },
-        "signatures": {TEST_USER_ID: {}},
-        "user_id": TEST_USER_ID,
-    }
-
-    device_data["signatures"][TEST_USER_ID][f"ed25519:{TEST_DEVICE_ID}"] = sign_json(
-        device_data, private_key
-    )
-
-    master_private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
-        MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES
-    )
-    b64_master_public_key = encode_base64(
-        master_private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
-    )
-
     print(
         f"""\
 /* Test data for cryptography tests
@@ -79,42 +83,213 @@ def main() -> None:
  * Do not edit by hand! This file is generated by `./generate-test-data.py`
  */
 
-import {{ IDeviceKeys }} from "../../../src/@types/crypto";
-import {{ IDownloadKeyResult }} from "../../../src";
+import {{ IDeviceKeys, IMegolmSessionData }} from "../../../src/@types/crypto";
+import {{ IDownloadKeyResult, IEvent }} from "../../../src";
+import {{ KeyBackupSession, KeyBackupInfo }} from "../../../src/crypto-api/keybackup";
 
 /* eslint-disable comma-dangle */
 
-export const TEST_USER_ID = "{TEST_USER_ID}";
-export const TEST_DEVICE_ID = "{TEST_DEVICE_ID}";
+// Alice data
 
-/** The base64-encoded public ed25519 key for this device */
-export const TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64 = "{b64_public_key}";
+{build_test_data(ALICE_DATA)}
+// Bob data
 
-/** Signed device data, suitable for returning from a `/keys/query` call */
-export const SIGNED_TEST_DEVICE_DATA: IDeviceKeys = {json.dumps(device_data, indent=4)};
-
-/** base64-encoded public master cross-signing key */
-export const MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "{b64_master_public_key}";
-
-/** Signed cross-signing keys data, also suitable for returning from a `/keys/query` call */
-export const SIGNED_CROSS_SIGNING_KEYS_DATA: Partial<IDownloadKeyResult> = {
-        json.dumps(build_cross_signing_keys_data(), indent=4)
-};
+{build_test_data(BOB_DATA, "BOB_")}
 """,
         end="",
     )
 
+# Use static seed to have stable random test data upon new generation
+seed(10)
 
-def build_cross_signing_keys_data() -> dict:
+def build_test_data(user_data, prefix = "") -> str:
+    private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
+             user_data["TEST_DEVICE_PRIVATE_KEY_BYTES"]
+        )
+
+    device_curve_key = x25519.X25519PrivateKey.from_private_bytes(
+             user_data["TEST_DEVICE_CURVE_PRIVATE_KEY_BYTES"]
+        )
+
+    b64_public_key = encode_base64(
+        private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+    )
+
+    device_data = {
+        "algorithms": ["m.olm.v1.curve25519-aes-sha2", "m.megolm.v1.aes-sha2"],
+        "device_id":  user_data["TEST_DEVICE_ID"],
+        "keys": {
+            f"curve25519:{user_data['TEST_DEVICE_ID']}": "F4uCNNlcbRvc7CfBz95ZGWBvY1ALniG1J8+6rhVoKS0",
+            f"ed25519:{user_data['TEST_DEVICE_ID']}": b64_public_key,
+        },
+        "signatures": {user_data['TEST_USER_ID']: {}},
+        "user_id": user_data["TEST_USER_ID"],
+    }
+
+    device_data["signatures"][user_data["TEST_USER_ID"]][f"ed25519:{user_data['TEST_DEVICE_ID']}"] = sign_json(
+        device_data, private_key
+    )
+
+    master_private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
+        user_data["MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES"]
+    )
+    b64_master_public_key = encode_base64(
+        master_private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+    )
+    b64_master_private_key = encode_base64(user_data["MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES"])
+
+    self_signing_private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
+        user_data["SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES"]
+    )
+    b64_self_signing_public_key = encode_base64(
+        self_signing_private_key.public_key().public_bytes(
+            Encoding.Raw, PublicFormat.Raw
+        )
+    )
+    b64_self_signing_private_key = encode_base64( user_data["SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES"])
+
+    user_signing_private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
+         user_data["USER_CROSS_SIGNING_PRIVATE_KEY_BYTES"]
+    )
+    b64_user_signing_public_key = encode_base64(
+        user_signing_private_key.public_key().public_bytes(
+            Encoding.Raw, PublicFormat.Raw
+        )
+    )
+    b64_user_signing_private_key = encode_base64(user_data["USER_CROSS_SIGNING_PRIVATE_KEY_BYTES"])
+
+    backup_decryption_key = x25519.X25519PrivateKey.from_private_bytes(
+        base64.b64decode(user_data["B64_BACKUP_DECRYPTION_KEY"])
+    )
+    b64_backup_public_key = encode_base64(
+        backup_decryption_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+    )
+
+    backup_data = {
+        "algorithm": "m.megolm_backup.v1.curve25519-aes-sha2",
+        "version": "1",
+        "auth_data": {
+            "public_key": b64_backup_public_key,
+        },
+    }
+    # sign with our device key
+    sig = sign_json(backup_data["auth_data"], private_key)
+    backup_data["auth_data"]["signatures"] = {
+        user_data["TEST_USER_ID"]: {f"ed25519:{user_data['TEST_DEVICE_ID']}": sig}
+    }
+
+    set_of_exported_room_keys = [build_exported_megolm_key(device_curve_key)[0], build_exported_megolm_key(device_curve_key)[0]]
+
+    additional_exported_room_key, additional_exported_ed_key = build_exported_megolm_key(device_curve_key)
+    ratcheted_exported_room_key = symetric_ratchet_step_of_megolm_key(additional_exported_room_key, additional_exported_ed_key)
+
+    otk_to_sign = {
+        "key": user_data['OTK']
+    }
+    # sign our public otk key with our device key
+    otk = sign_json(otk_to_sign, private_key)
+    otks = {
+        user_data["TEST_USER_ID"]: {
+            user_data['TEST_DEVICE_ID']: {
+                 "signed_curve25519:AAAAHQ": {
+                    "key": user_data["OTK"],
+                    "signatures": {
+                        user_data["TEST_USER_ID"]: {f"ed25519:{user_data['TEST_DEVICE_ID']}": otk}
+                    }
+                 }
+            }
+        }
+    }
+
+
+    backed_up_room_key = encrypt_megolm_key_for_backup(additional_exported_room_key, backup_decryption_key.public_key())
+
+    clear_event, encrypted_event = generate_encrypted_event_content(additional_exported_room_key, additional_exported_ed_key, device_curve_key)
+
+    backup_recovery_key = export_recovery_key(user_data["B64_BACKUP_DECRYPTION_KEY"])
+
+    return f"""\
+export const {prefix}TEST_USER_ID = "{user_data['TEST_USER_ID']}";
+export const {prefix}TEST_DEVICE_ID = "{user_data['TEST_DEVICE_ID']}";
+export const {prefix}TEST_ROOM_ID = "{user_data['TEST_ROOM_ID']}";
+
+/** The base64-encoded public ed25519 key for this device */
+export const {prefix}TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64 = "{b64_public_key}";
+
+/** Signed device data, suitable for returning from a `/keys/query` call */
+export const {prefix}SIGNED_TEST_DEVICE_DATA: IDeviceKeys = {json.dumps(device_data, indent=4)};
+
+/** base64-encoded public master cross-signing key */
+export const {prefix}MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "{b64_master_public_key}";
+
+/** base64-encoded private master cross-signing key */
+export const {prefix}MASTER_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "{b64_master_private_key}";
+
+/** base64-encoded public self cross-signing key */
+export const {prefix}SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "{b64_self_signing_public_key}";
+
+/** base64-encoded private self signing cross-signing key */
+export const {prefix}SELF_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "{b64_self_signing_private_key}";
+
+/** base64-encoded public user cross-signing key */
+export const {prefix}USER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "{b64_user_signing_public_key}";
+
+/** base64-encoded private user signing cross-signing key */
+export const {prefix}USER_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "{b64_user_signing_private_key}";
+
+/** Signed cross-signing keys data, also suitable for returning from a `/keys/query` call */
+export const {prefix}SIGNED_CROSS_SIGNING_KEYS_DATA: Partial<IDownloadKeyResult> = {
+        json.dumps(build_cross_signing_keys_data(user_data), indent=4)
+};
+
+/** Signed OTKs, returned by `POST /keys/claim` */
+export const {prefix}ONE_TIME_KEYS = { json.dumps(otks, indent=4) };
+
+/** base64-encoded backup decryption (private) key */
+export const {prefix}BACKUP_DECRYPTION_KEY_BASE64 = "{ user_data['B64_BACKUP_DECRYPTION_KEY'] }";
+
+/** Backup decryption key in export format */
+export const {prefix}BACKUP_DECRYPTION_KEY_BASE58 = "{ backup_recovery_key }";
+
+/** Signed backup data, suitable for return from `GET /_matrix/client/v3/room_keys/keys/{{roomId}}/{{sessionId}}` */
+export const {prefix}SIGNED_BACKUP_DATA: KeyBackupInfo = { json.dumps(backup_data, indent=4) };
+
+/** A set of megolm keys that can be imported via CryptoAPI#importRoomKeys */
+export const {prefix}MEGOLM_SESSION_DATA_ARRAY: IMegolmSessionData[] = {
+    json.dumps(set_of_exported_room_keys, indent=4)
+};
+
+/** An exported megolm session */
+export const {prefix}MEGOLM_SESSION_DATA: IMegolmSessionData = {
+        json.dumps(additional_exported_room_key, indent=4)
+};
+
+/** A ratcheted version of {prefix}MEGOLM_SESSION_DATA */
+export const {prefix}RATCHTED_MEGOLM_SESSION_DATA: IMegolmSessionData = {
+        json.dumps(ratcheted_exported_room_key, indent=4)
+};
+
+/** The key from {prefix}MEGOLM_SESSION_DATA, encrypted for backup using `m.megolm_backup.v1.curve25519-aes-sha2` algorithm*/
+export const {prefix}CURVE25519_KEY_BACKUP_DATA: KeyBackupSession = {json.dumps(backed_up_room_key, indent=4)};
+
+/** A test clear event */
+export const {prefix}CLEAR_EVENT: Partial<IEvent> = {json.dumps(clear_event, indent=4)};
+
+/** The encrypted CLEAR_EVENT by MEGOLM_SESSION_DATA */
+export const {prefix}ENCRYPTED_EVENT: Partial<IEvent> = {json.dumps(encrypted_event, indent=4)};
+"""
+
+
+def build_cross_signing_keys_data(user_data) -> dict:
     """Build the signed cross-signing-keys data for return from /keys/query"""
     master_private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
-        MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES
+        user_data["MASTER_CROSS_SIGNING_PRIVATE_KEY_BYTES"]
     )
     b64_master_public_key = encode_base64(
         master_private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
     )
     self_signing_private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
-        SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES
+         user_data["SELF_CROSS_SIGNING_PRIVATE_KEY_BYTES"]
     )
     b64_self_signing_public_key = encode_base64(
         self_signing_private_key.public_key().public_bytes(
@@ -122,7 +297,7 @@ def build_cross_signing_keys_data() -> dict:
         )
     )
     user_signing_private_key = ed25519.Ed25519PrivateKey.from_private_bytes(
-        USER_CROSS_SIGNING_PRIVATE_KEY_BYTES
+         user_data["USER_CROSS_SIGNING_PRIVATE_KEY_BYTES"]
     )
     b64_user_signing_public_key = encode_base64(
         user_signing_private_key.public_key().public_bytes(
@@ -132,39 +307,39 @@ def build_cross_signing_keys_data() -> dict:
     # create without signatures initially
     cross_signing_keys_data = {
         "master_keys": {
-            TEST_USER_ID: {
+             user_data["TEST_USER_ID"]: {
                 "keys": {
                     f"ed25519:{b64_master_public_key}": b64_master_public_key,
                 },
-                "user_id": TEST_USER_ID,
+                "user_id": user_data["TEST_USER_ID"],
                 "usage": ["master"],
             }
         },
         "self_signing_keys": {
-            TEST_USER_ID: {
+            user_data["TEST_USER_ID"]: {
                 "keys": {
                     f"ed25519:{b64_self_signing_public_key}": b64_self_signing_public_key,
                 },
-                "user_id": TEST_USER_ID,
+                "user_id": user_data["TEST_USER_ID"],
                 "usage": ["self_signing"],
             },
         },
         "user_signing_keys": {
-            TEST_USER_ID: {
+            user_data["TEST_USER_ID"]: {
                 "keys": {
                     f"ed25519:{b64_user_signing_public_key}": b64_user_signing_public_key,
                 },
-                "user_id": TEST_USER_ID,
+                "user_id": user_data["TEST_USER_ID"],
                 "usage": ["user_signing"],
             },
         },
     }
     # sign the sub-keys with the master
     for k in ["self_signing_keys", "user_signing_keys"]:
-        to_sign = cross_signing_keys_data[k][TEST_USER_ID]
+        to_sign = cross_signing_keys_data[k][user_data["TEST_USER_ID"]]
         sig = sign_json(to_sign, master_private_key)
         to_sign["signatures"] = {
-            TEST_USER_ID: {f"ed25519:{b64_master_public_key}": sig}
+            user_data["TEST_USER_ID"]: {f"ed25519:{b64_master_public_key}": sig}
         }
 
     return cross_signing_keys_data
@@ -198,6 +373,282 @@ def sign_json(json_object: dict, private_key: ed25519.Ed25519PrivateKey) -> str:
 
     return signature_base64
 
+def build_exported_megolm_key(device_curve_key: x25519.X25519PrivateKey) -> tuple[dict, ed25519.Ed25519PrivateKey]:
+    """
+    Creates an exported megolm room key, as per https://gitlab.matrix.org/matrix-org/olm/blob/master/docs/megolm.md#session-export-format
+    that can be imported via importRoomKeys API.
+    Returns the exported key, the matching privat edKey (needed to encrypt)
+    """
+    index = 0
+    private_key = ed25519.Ed25519PrivateKey.from_private_bytes(randbytes(32))
+    # Just use radom bytes for the ratchet parts
+    ratchet = randbytes(32 * 4)
+    # exported key, start with version byte
+    exported_key = bytearray(b'\x01')
+    exported_key += index.to_bytes(4, 'big')
+    exported_key += ratchet
+    # KPub
+    exported_key += private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+
+
+    megolm_export = {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "room_id": "!room:id",
+        "sender_key": encode_base64(
+            device_curve_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+        ),
+        "session_id": encode_base64(
+            private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+        ),
+        "session_key": encode_base64(exported_key),
+        "sender_claimed_keys": {
+            "ed25519": encode_base64(ed25519.Ed25519PrivateKey.from_private_bytes(randbytes(32)).public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)),
+        },
+        "forwarding_curve25519_key_chain": [],
+    }
+
+    return megolm_export, private_key
+
+def symetric_ratchet_step_of_megolm_key(previous: dict , megolm_private_key: ed25519.Ed25519PrivateKey) -> dict:
+
+    """
+    Very simple ratchet step from 0 to 1
+    Used to generate a ratcheted key to test unknown message index.
+    """
+    session_key: str = previous["session_key"]
+
+    # Get the megolm R0 from the export format
+    decoded = base64.b64decode(session_key.encode("ascii"))
+    ri = decoded[5:133]
+
+    ri0 = ri[0:32]
+    ri1 = ri[32:64]
+    ri2 = ri[64:96]
+    ri3 = ri[96:128]
+
+    h = hmac.HMAC(ri3, hashes.SHA256())
+    h.update(b'x\03')
+    ri1_3 = h.finalize()
+
+    index = 1
+    private_key = megolm_private_key
+
+    # exported key, start with version byte
+    exported_key = bytearray(b'\x01')
+    exported_key += index.to_bytes(4, 'big')
+    exported_key += ri0
+    exported_key += ri1
+    exported_key += ri2
+    exported_key += ri1_3
+    # KPub
+    exported_key += private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)
+
+
+    megolm_export = {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "room_id": "!room:id",
+        "sender_key": previous["sender_key"],
+        "session_id": previous["session_id"],
+        "session_key": encode_base64(exported_key),
+        "sender_claimed_keys": previous["sender_claimed_keys"],
+        "forwarding_curve25519_key_chain": [],
+    }
+
+    return megolm_export
+
+def encrypt_megolm_key_for_backup(session_data: dict, backup_public_key: x25519.X25519PublicKey) -> dict:
+
+    """
+    Encrypts an exported megolm key for key backup, using the m.megolm_backup.v1.curve25519-aes-sha2 algorithm.
+    """
+    data = encode_canonical_json(session_data)
+
+    # Generate an ephemeral curve25519 key, and perform an ECDH with the ephemeral key
+    # and the backup’s public key to generate a shared secret.
+    # The public half of the ephemeral key, encoded using unpadded base64,
+    # becomes the ephemeral property of the session_data.
+    ephemeral_keypair = x25519.X25519PrivateKey.from_private_bytes(randbytes(32))
+    shared_secret = ephemeral_keypair.exchange(backup_public_key)
+    ephemeral = encode_base64(ephemeral_keypair.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw))
+
+    # Using the shared secret, generate 80 bytes by performing an HKDF using SHA-256 as the hash,
+    # with a salt of 32 bytes of 0, and with the empty string as the info.
+    # The first 32 bytes are used as the AES key, the next 32 bytes are used as the MAC key,
+    #  and the last 16 bytes are used as the AES initialization vector.
+    salt = bytes(32)
+    info = b""
+
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=80,
+        salt=salt,
+        info=info,
+    )
+
+    raw_key = hkdf.derive(shared_secret)
+    aes_key = raw_key[:32]
+    mac = raw_key[32:64]
+    iv = raw_key[64:80]
+
+    # Stringify the JSON object, and encrypt it using AES-CBC-256 with PKCS#7 padding.
+    # This encrypted data, encoded using unpadded base64, becomes the ciphertext property of the session_data.
+    cipher = Cipher(algorithms.AES(aes_key), modes.CBC(iv))
+    encryptor = cipher.encryptor()
+    padder = padding.PKCS7(128).padder()
+    padded_data = padder.update(data) + padder.finalize()
+    ct = encryptor.update(padded_data) + encryptor.finalize()
+    cipher_text = encode_base64(ct)
+
+    # Pass the raw encrypted data (prior to base64 encoding) through HMAC-SHA-256 using the MAC key generated above.
+    # The first 8 bytes of the resulting MAC are base64-encoded, and become the mac property of the session_data.
+    h = hmac.HMAC(mac, hashes.SHA256())
+    # h.update(ct)
+    signature = h.finalize()
+    mac = encode_base64(signature[:8])
+
+    encrypted_key = {
+        "first_message_index": 1,
+        "forwarded_count": 0,
+        "is_verified": False,
+        "session_data": {
+            "ciphertext": cipher_text,
+            "ephemeral": ephemeral,
+            "mac": mac
+        }
+
+    }
+
+    return encrypted_key
+
+def generate_encrypted_event_content(exported_key: dict, ed_key: ed25519.Ed25519PrivateKey, curve_key: x25519.X25519PrivateKey) -> tuple[dict, dict]:
+    """
+        Encrypts an event using the given key in session export format.
+        Will not do any ratcheting, just encrypt at index 0.
+    """
+
+    clear_event = {
+        "type": "m.room.message",
+        "room_id": "!room:id",
+        "sender": "@alice:localhost",
+        "content": {
+            "msgtype": "m.text",
+            "body": "Hello world"
+        }
+    }
+
+    session_key: str = exported_key["session_key"]
+
+    # Get the megolm R0 from the export format
+    decoded = base64.b64decode(session_key.encode("ascii"))
+    r0 = decoded[5:133]
+
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=80,
+        salt=bytes(32),
+        info=b"MEGOLM_KEYS",
+    )
+
+    raw_key = hkdf.derive(r0)
+    aes_key = raw_key[:32]
+    mac = raw_key[32:64]
+    aes_iv = raw_key[64:80]
+
+    payload_json = {
+        "room_id": clear_event["room_id"],
+        "type": clear_event["type"],
+        "content": clear_event["content"]
+    }
+
+    payload_string = encode_canonical_json(payload_json)
+
+    cipher = Cipher(algorithms.AES(aes_key), modes.CBC(aes_iv))
+    encryptor = cipher.encryptor()
+    padder = padding.PKCS7(128).padder()
+
+    padded_data = padder.update(payload_string)
+    padded_data += padder.finalize()
+
+    ct = encryptor.update(padded_data) + encryptor.finalize()
+
+    # The ratchet index i, and the cipher-text, are then packed
+    # into a message as described in Message format. Then the entire message
+    # (including the version bytes and all payload bytes) are passed through
+    # HMAC-SHA-256. The first 8 bytes of the MAC are appended to the message.
+    message = bytearray()
+    message += b'\x03'
+    # int tag for index
+    message += b'\x08'
+    # index is 0
+    message += b'\x00'
+    message += b'\x12'
+    # probably works only for short messages
+    message += len(ct).to_bytes(1, 'big')
+    # encrypted data
+    message += ct
+
+    h = hmac.HMAC(mac, hashes.SHA256())
+    h.update(message)
+    signature = h.finalize()
+    mac = signature[:8]
+
+    message += mac
+
+    # Finally, the authenticated message is signed using the Ed25519 keypair;
+    # the 64 byte signature is appended to the message
+    signature = ed_key.sign(bytes(message))
+
+    message += signature
+
+    cipher_text = encode_base64(message)
+
+    encrypted_payload = {
+        "algorithm" : "m.megolm.v1.aes-sha2",
+        "sender_key" : encode_base64(curve_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw)),
+        "ciphertext" : cipher_text,
+        "session_id" : exported_key["session_id"],
+        "device_id" : "TEST_DEVICE"
+    }
+
+    encrypted_event = {
+        "type": "m.room.encrypted",
+        "room_id": "!room:id",
+        "sender": "@alice:localhost",
+        "content": encrypted_payload,
+        "event_id": "$event1",
+        "origin_server_ts": 1507753886000,
+    }
+
+    return clear_event, encrypted_event
+
+
+def export_recovery_key(key_b64: str) -> str:
+    """
+        Export a private recovery key as a recovery key that can be presented to users.
+        As per spec https://spec.matrix.org/v1.8/client-server-api/#recovery-key
+    """
+    private_key_bytes = base64.b64decode(key_b64)
+
+    # The 256-bit curve25519 private key is prepended by the bytes 0x8B and 0x01
+    export_bytes = bytearray()
+    export_bytes += b'\x8b'
+    export_bytes += b'\x01'
+
+    export_bytes += private_key_bytes
+
+    # All the bytes in the string above, including the two header bytes,
+    # are XORed together to form a parity byte. This parity byte is appended to the byte string.
+    parity_byte = 0 #b'\x8b' ^ b'\x01'
+    [parity_byte := parity_byte ^ x for x in export_bytes]
+
+    export_bytes += parity_byte.to_bytes(1, 'big')
+
+    # The byte string is encoded using base58
+    recovery_key = base58.b58encode(export_bytes).decode('utf-8')
+
+    split = [recovery_key[i:i + 4] for i in range(0, len(recovery_key), 4)]
+    return ' '.join(split)
+
 
 if __name__ == "__main__":
     main()

spec/test-utils/test-data/index.ts

@@ -3,13 +3,17 @@
  * Do not edit by hand! This file is generated by `./generate-test-data.py`
  */
 
-import { IDeviceKeys } from "../../../src/@types/crypto";
-import { IDownloadKeyResult } from "../../../src";
+import { IDeviceKeys, IMegolmSessionData } from "../../../src/@types/crypto";
+import { IDownloadKeyResult, IEvent } from "../../../src";
+import { KeyBackupSession, KeyBackupInfo } from "../../../src/crypto-api/keybackup";
 
 /* eslint-disable comma-dangle */
 
+// Alice data
+
 export const TEST_USER_ID = "@alice:localhost";
 export const TEST_DEVICE_ID = "test_device";
+export const TEST_ROOM_ID = "!room:id";
 
 /** The base64-encoded public ed25519 key for this device */
 export const TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64 = "YI/7vbGVLpGdYtuceQR8MSsKB/QjgfMXM1xqnn+0NWU";
@@ -36,6 +40,21 @@ export const SIGNED_TEST_DEVICE_DATA: IDeviceKeys = {
 /** base64-encoded public master cross-signing key */
 export const MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "J+5An10v1vzZpAXTYFokD1/PEVccFnLC61EfRXit0UY";
 
+/** base64-encoded private master cross-signing key */
+export const MASTER_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "ZG95b3VzcGVha3doYWFhYWFhYWFhYWFhYWFhYWFhbGU";
+
+/** base64-encoded public self cross-signing key */
+export const SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "aU2+2CyXQTCuDcmWW0EL2bhJ6PdjFW2LbAsbHqf02AY";
+
+/** base64-encoded private self signing cross-signing key */
+export const SELF_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "c2VsZnNlbGZzZWxmc2VsZnNlbGZzZWxmc2VsZnNlbGY";
+
+/** base64-encoded public user cross-signing key */
+export const USER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "g5TC/zjQXyZYuDLZv7a41z5fFVrXpYPypG//AFQj8hY";
+
+/** base64-encoded private user signing cross-signing key */
+export const USER_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "dXNlcnVzZXJ1c2VydXNlcnVzZXJ1c2VydXNlcnVzZXI";
+
 /** Signed cross-signing keys data, also suitable for returning from a `/keys/query` call */
 export const SIGNED_CROSS_SIGNING_KEYS_DATA: Partial<IDownloadKeyResult> = {
     "master_keys": {
@@ -82,3 +101,351 @@ export const SIGNED_CROSS_SIGNING_KEYS_DATA: Partial<IDownloadKeyResult> = {
         }
     }
 };
+
+/** Signed OTKs, returned by `POST /keys/claim` */
+export const ONE_TIME_KEYS = {
+    "@alice:localhost": {
+        "test_device": {
+            "signed_curve25519:AAAAHQ": {
+                "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw",
+                "signatures": {
+                    "@alice:localhost": {
+                        "ed25519:test_device": "25djC6Rk6gIgFBMVawY9X9LnY8XMMziey6lKqL8Q5Bbp7T1vw9uk0RE7eKO2a/jNLcYroO2xRztGhBrKz5sOCQ"
+                    }
+                }
+            }
+        }
+    }
+};
+
+/** base64-encoded backup decryption (private) key */
+export const BACKUP_DECRYPTION_KEY_BASE64 = "dwdtCnMYpX08FsFyUbJmRd9ML4frwJkqsXf7pR25LCo=";
+
+/** Backup decryption key in export format */
+export const BACKUP_DECRYPTION_KEY_BASE58 = "EsTc LW2K PGiF wKEA 3As5 g5c4 BXwk qeeJ ZJV8 Q9fu gUMN UE4d";
+
+/** Signed backup data, suitable for return from `GET /_matrix/client/v3/room_keys/keys/{roomId}/{sessionId}` */
+export const SIGNED_BACKUP_DATA: KeyBackupInfo = {
+    "algorithm": "m.megolm_backup.v1.curve25519-aes-sha2",
+    "version": "1",
+    "auth_data": {
+        "public_key": "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+        "signatures": {
+            "@alice:localhost": {
+                "ed25519:test_device": "KDSNeumirTsd8piI0oVfv/wzg4J4HlEc7rs5XhODFcJ/YAcUdg65ajsZG+rLI0TQOSSGjorJqcrSiSB1HRSCAA"
+            }
+        }
+    }
+};
+
+/** A set of megolm keys that can be imported via CryptoAPI#importRoomKeys */
+export const MEGOLM_SESSION_DATA_ARRAY: IMegolmSessionData[] = [
+    {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "room_id": "!room:id",
+        "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
+        "session_id": "FYOoKQSwe4d9jhTZ/LQCZFJINjPEqZ7Or4Z08reP92M",
+        "session_key": "AQAAAABZ0jXQOprFfXe41tIFmAtHxflJp4O2hM/vzQQpOazOCFeWSoW5P3Z9Q+voU3eXehMwyP8/hm/Q8xLP6/PmJdy+71se/17kdFwcDGgLxBWfa4ODM9zlI4EjKbNqmiii5loJ7rBhA/XXaw80m0hfU6zTDX/KrO55J0Pt4vJ0LDa3LBWDqCkEsHuHfY4U2fy0AmRSSDYzxKmezq+GdPK3j/dj",
+        "sender_claimed_keys": {
+            "ed25519": "QdgHgdpDgihgovpPzUiThXur1fbErTFh7paFvNKSgN0"
+        },
+        "forwarding_curve25519_key_chain": []
+    },
+    {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "room_id": "!room:id",
+        "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
+        "session_id": "mPYSGA2l1tOQiipEDEVYhDSdTSFh2lDW1qpGKYZRxTc",
+        "session_key": "AQAAAAAHwgkB49BTPAEGTCK6degxUIbl8GPG2ugPRYhNtOpNic63u11+baXFfjDw5fmVfD1gJXpQQjGsqrIYioxrB1xzl7mfb942UHhYdaMQZowpp1fSpJVsxR5TddUU2EWifYD9EQsoz8mY1zqoazm4vUP4v9yxaTcUBj2c6HMJCY0gCJj2EhgNpdbTkIoqRAxFWIQ0nU0hYdpQ1taqRimGUcU3",
+        "sender_claimed_keys": {
+            "ed25519": "IrkbT6H+0urDf6wKDSyVC1fh1t84Vz6T62snni86Cog"
+        },
+        "forwarding_curve25519_key_chain": []
+    }
+];
+
+/** An exported megolm session */
+export const MEGOLM_SESSION_DATA: IMegolmSessionData = {
+    "algorithm": "m.megolm.v1.aes-sha2",
+    "room_id": "!room:id",
+    "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
+    "session_id": "ipdI6Zs/7DzFTEhiA2iGaMDfHkIYCleqXT6L+5e1/co",
+    "session_key": "AQAAAABXGO+Z9jlQJhIL6ByhXrv2BwCIxkhh7MXpKLsYmXkJcWrQlirmXmD79ga1zo+I4DCtEZzyGSpDWXBC6G7ez3H4gDMBam1RE3Jm5tc+oTlIri32UkYgSL0kBkcEnttqmIXBlK8tAfJo3cJnlh7F4ltEOAqrdME6dU0zXTkqXmURqYqXSOmbP+w8xUxIYgNohmjA3x5CGApXql0+i/uXtf3K",
+    "sender_claimed_keys": {
+        "ed25519": "Bhbpt6hqMZlSH4sJV7xiEEEiPVeTWz4Vkujl1EMdIPI"
+    },
+    "forwarding_curve25519_key_chain": []
+};
+
+/** A ratcheted version of MEGOLM_SESSION_DATA */
+export const RATCHTED_MEGOLM_SESSION_DATA: IMegolmSessionData = {
+    "algorithm": "m.megolm.v1.aes-sha2",
+    "room_id": "!room:id",
+    "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
+    "session_id": "ipdI6Zs/7DzFTEhiA2iGaMDfHkIYCleqXT6L+5e1/co",
+    "session_key": "AQAAAAFXGO+Z9jlQJhIL6ByhXrv2BwCIxkhh7MXpKLsYmXkJcWrQlirmXmD79ga1zo+I4DCtEZzyGSpDWXBC6G7ez3H4gDMBam1RE3Jm5tc+oTlIri32UkYgSL0kBkcEnttqmIUWvpwC7by/yg231+gyzu9lDHAU4ivCj48pt7WGiORWmIqXSOmbP+w8xUxIYgNohmjA3x5CGApXql0+i/uXtf3K",
+    "sender_claimed_keys": {
+        "ed25519": "Bhbpt6hqMZlSH4sJV7xiEEEiPVeTWz4Vkujl1EMdIPI"
+    },
+    "forwarding_curve25519_key_chain": []
+};
+
+/** The key from MEGOLM_SESSION_DATA, encrypted for backup using `m.megolm_backup.v1.curve25519-aes-sha2` algorithm*/
+export const CURVE25519_KEY_BACKUP_DATA: KeyBackupSession = {
+    "first_message_index": 1,
+    "forwarded_count": 0,
+    "is_verified": false,
+    "session_data": {
+        "ciphertext": "r6HRk2/Im2yJe5cLP8R81aVjFWjYWPHpw7TVxphiSK1cdIDZTTK57r6MfU+0i/mTPn+/PosT74OvYwCnehy2d1BPGxhDl8AhPcBu3//Kzlq2o5CssPsw+88gRehkAsPg9Zp5G9sL9to6giltvTWTbsaQpmvv3HLmBOYSFIxvyZrOT/Ffqu325f0IEsKcyV2BdIkw8Ob9Xt+VWoe4MYEGG6y1T8W125zeFgKWI4Ow76uput64H9zZjIo+Cc+hCTO9Ea4EnosSjizCotevkNck7C/zGgfhBikiohROb6SbaZgxicSsEDZ+f7brnri9yP3iXS3PMDHHpa1+XzG2VOG/Y9OQZpkPq+pbLrCC+NWJeJPslDAK5i+RURwzjnPmaHKCRHTq86CwhFyiCDf61MGwCY3xjrmBJg44BCdxWqCx0YJvwsvVqqnl4vTieUfrwThNPsQ81aVkDHvlmrgrTt8icDa8jTJhu34jem+pbRSEM5aJikV4B+zYiLz+dH/v6UpYA2eG8ReOvwpPXp6CAcIlplRPpWbMBeLFVcPkT4KAXTp9exFpB4on4pf8OsaDomlt4qAA0rhAZmhPWPKcU/A0Tz4gyMu54OivVtw1SPj+5Iq+YDQ8jB6Po3ApzMf6fwF9x/FjevbboFB05X2Jr0NrbFqXMOUwXHMgDAGiIWX8+gkmmbaiNWqg2etjN94pobQSGZelb18XGN7kuwMk+Zwk7A",
+        "ephemeral": "q+P1WdRtEiPIEtNuuGrRcueZxUbLnSKdsuTAkxewXgU",
+        "mac": "OibmACbORhI"
+    }
+};
+
+/** A test clear event */
+export const CLEAR_EVENT: Partial<IEvent> = {
+    "type": "m.room.message",
+    "room_id": "!room:id",
+    "sender": "@alice:localhost",
+    "content": {
+        "msgtype": "m.text",
+        "body": "Hello world"
+    }
+};
+
+/** The encrypted CLEAR_EVENT by MEGOLM_SESSION_DATA */
+export const ENCRYPTED_EVENT: Partial<IEvent> = {
+    "type": "m.room.encrypted",
+    "room_id": "!room:id",
+    "sender": "@alice:localhost",
+    "content": {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "sender_key": "WimPd2udAU/1S/+YBpPbmr9L+0H5H+BnAVHSwDxlPGc",
+        "ciphertext": "AwgAEnAkBmciEAyhh1j6DCk29UXJ7kv/kvayUNfuNT0iAioLxcXjFXOZ5ho3jF1/wrytlt0Lb298uMM67OxdVMi+/mMfYpwlvi07P9cIH6CMSj8tyhYoWl0SrKY6tkPf5GWOlRSRRKbziXa96FHXvnA3V2FCAIGtAe3G4ei5RPbhkmKAFBLAen33/D6MjJVqU8Ojr5vTkgls5eyirarlVpsmnH06alDaxO8avrU0NL+Vsw26xvlUQgEMOnUJ",
+        "session_id": "ipdI6Zs/7DzFTEhiA2iGaMDfHkIYCleqXT6L+5e1/co",
+        "device_id": "TEST_DEVICE"
+    },
+    "event_id": "$event1",
+    "origin_server_ts": 1507753886000
+};
+
+// Bob data
+
+export const BOB_TEST_USER_ID = "@bob:xyz";
+export const BOB_TEST_DEVICE_ID = "bob_device";
+export const BOB_TEST_ROOM_ID = "!room:id";
+
+/** The base64-encoded public ed25519 key for this device */
+export const BOB_TEST_DEVICE_PUBLIC_ED25519_KEY_BASE64 = "jmY0h8QS6Te6gxyjOmMc0eKOqmbAtXpVo4CCWFubk50";
+
+/** Signed device data, suitable for returning from a `/keys/query` call */
+export const BOB_SIGNED_TEST_DEVICE_DATA: IDeviceKeys = {
+    "algorithms": [
+        "m.olm.v1.curve25519-aes-sha2",
+        "m.megolm.v1.aes-sha2"
+    ],
+    "device_id": "bob_device",
+    "keys": {
+        "curve25519:bob_device": "F4uCNNlcbRvc7CfBz95ZGWBvY1ALniG1J8+6rhVoKS0",
+        "ed25519:bob_device": "jmY0h8QS6Te6gxyjOmMc0eKOqmbAtXpVo4CCWFubk50"
+    },
+    "user_id": "@bob:xyz",
+    "signatures": {
+        "@bob:xyz": {
+            "ed25519:bob_device": "4ApBs9jaeGyfdYaWRUdBvQAkDyXjACJ9KJ0xLHMgiFT/1yo6VqPTx2iziKGnrBiGhbtKNxEhDPOvZZkBU73cDQ"
+        }
+    }
+};
+
+/** base64-encoded public master cross-signing key */
+export const BOB_MASTER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "KKVOHOB2LsW7hFJwqyzXpA+vp7u5+gaMWUJvBS7mjuA";
+
+/** base64-encoded private master cross-signing key */
+export const BOB_MASTER_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "RG95b3VzcGVha3doYWFhYWFhYWFhYWFhYWFhYWFhbGU";
+
+/** base64-encoded public self cross-signing key */
+export const BOB_SELF_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "DaScI3WulBvDjf/d2vdyP5Cgjdypn1c/PSDX23MgN+A";
+
+/** base64-encoded private self signing cross-signing key */
+export const BOB_SELF_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "U2VsZnNlbGZzZWxmc2VsZnNlbGZzZWxmc2VsZnNlbGY";
+
+/** base64-encoded public user cross-signing key */
+export const BOB_USER_CROSS_SIGNING_PUBLIC_KEY_BASE64 = "lXP89FP6zvFH9TSbU1S8uSdAsVawm1NmV6z+Rfr3lEw";
+
+/** base64-encoded private user signing cross-signing key */
+export const BOB_USER_CROSS_SIGNING_PRIVATE_KEY_BASE64 = "VXNlcnVzZXJ1c2VydXNlcnVzZXJ1c2VydXNlcnVzZXI";
+
+/** Signed cross-signing keys data, also suitable for returning from a `/keys/query` call */
+export const BOB_SIGNED_CROSS_SIGNING_KEYS_DATA: Partial<IDownloadKeyResult> = {
+    "master_keys": {
+        "@bob:xyz": {
+            "keys": {
+                "ed25519:KKVOHOB2LsW7hFJwqyzXpA+vp7u5+gaMWUJvBS7mjuA": "KKVOHOB2LsW7hFJwqyzXpA+vp7u5+gaMWUJvBS7mjuA"
+            },
+            "user_id": "@bob:xyz",
+            "usage": [
+                "master"
+            ]
+        }
+    },
+    "self_signing_keys": {
+        "@bob:xyz": {
+            "keys": {
+                "ed25519:DaScI3WulBvDjf/d2vdyP5Cgjdypn1c/PSDX23MgN+A": "DaScI3WulBvDjf/d2vdyP5Cgjdypn1c/PSDX23MgN+A"
+            },
+            "user_id": "@bob:xyz",
+            "usage": [
+                "self_signing"
+            ],
+            "signatures": {
+                "@bob:xyz": {
+                    "ed25519:KKVOHOB2LsW7hFJwqyzXpA+vp7u5+gaMWUJvBS7mjuA": "RxM8iJU6ZkyzQSVtNnXIJMPyEahVsN+fQQTBNKAs+kqySFyXBgchx+8czZaAhJCpXh9gD1nskT4yyFd2eyUXBw"
+                }
+            }
+        }
+    },
+    "user_signing_keys": {
+        "@bob:xyz": {
+            "keys": {
+                "ed25519:lXP89FP6zvFH9TSbU1S8uSdAsVawm1NmV6z+Rfr3lEw": "lXP89FP6zvFH9TSbU1S8uSdAsVawm1NmV6z+Rfr3lEw"
+            },
+            "user_id": "@bob:xyz",
+            "usage": [
+                "user_signing"
+            ],
+            "signatures": {
+                "@bob:xyz": {
+                    "ed25519:KKVOHOB2LsW7hFJwqyzXpA+vp7u5+gaMWUJvBS7mjuA": "jF8fvnPZulrPyh/4E8dNDVBP3iHHl9bRc+rRArVyGzoom+uVrokOck7BN2YmPyCRFZJJx7fgRA1Bveyu+mTVAg"
+                }
+            }
+        }
+    }
+};
+
+/** Signed OTKs, returned by `POST /keys/claim` */
+export const BOB_ONE_TIME_KEYS = {
+    "@bob:xyz": {
+        "bob_device": {
+            "signed_curve25519:AAAAHQ": {
+                "key": "j3fR3HemM16M7CWhoI4Sk5ZsdmdfQHsKL1xuSft6MSw",
+                "signatures": {
+                    "@bob:xyz": {
+                        "ed25519:bob_device": "dlZc9VA/hP980Mxvu9qwi0qJx8VK7sADGOM48CE01YM7K/Mbty9lis/QjtQAWqDg371QyynVRjEzt9qj7eSFCg"
+                    }
+                }
+            }
+        }
+    }
+};
+
+/** base64-encoded backup decryption (private) key */
+export const BOB_BACKUP_DECRYPTION_KEY_BASE64 = "DwdtCnMYpX08FsFyUbJmRd9ML4frwJkqsXf7pR25LCo=";
+
+/** Backup decryption key in export format */
+export const BOB_BACKUP_DECRYPTION_KEY_BASE58 = "EsT5 Sd5m mEXs NQYE ibRe 3q9E 4aXW rHih 5f9J 6rU6 AfwY mASR";
+
+/** Signed backup data, suitable for return from `GET /_matrix/client/v3/room_keys/keys/{roomId}/{sessionId}` */
+export const BOB_SIGNED_BACKUP_DATA: KeyBackupInfo = {
+    "algorithm": "m.megolm_backup.v1.curve25519-aes-sha2",
+    "version": "1",
+    "auth_data": {
+        "public_key": "ZRuVWcWlDuvOwZRygccUCD4Avtnt130800I+WQNwwRY",
+        "signatures": {
+            "@bob:xyz": {
+                "ed25519:bob_device": "lDIMj3VC0WazE2FamGHpmbiqKf9Z4pO4qapZ5TL5BnD3c+dvb+2waOEd6pgay/pmrQ6MW4Eu2KDEpe1fnHc3BA"
+            }
+        }
+    }
+};
+
+/** A set of megolm keys that can be imported via CryptoAPI#importRoomKeys */
+export const BOB_MEGOLM_SESSION_DATA_ARRAY: IMegolmSessionData[] = [
+    {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "room_id": "!room:id",
+        "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+        "session_id": "/2K+V777vipCxPZ0gpY9qcpz1DYaXwuMRIu0UEP0Wa0",
+        "session_key": "AQAAAAAclzWVMeWBKH+B/WMowa3rb4ma3jEl6n5W4GCs9ue65CruzD3ihX+85pZ9hsV9Bf6fvhjp76WNRajoJYX0UIt7aosjmu0i+H+07hEQ0zqTKpVoSH0ykJ6stAMhdr6Q4uW5crBmdTTBIsqmoWsNJZKKoE2+ldYrZ1lrFeaJbjBIY/9ivle++74qQsT2dIKWPanKc9Q2Gl8LjESLtFBD9Fmt",
+        "sender_claimed_keys": {
+            "ed25519": "F4P7f1Z0RjbiZMgHk1xBCG3KC4/Ng9PmxLJ4hQ13sHA"
+        },
+        "forwarding_curve25519_key_chain": []
+    },
+    {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "room_id": "!room:id",
+        "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+        "session_id": "+07YOpSgdZ1X9le3n3NMByw0V1B0H0Djnbm76jgmWoo",
+        "session_key": "AQAAAAAjWfIMo9+BWS8IvhfsQuomxXXXGy11tJs0ej505xxd1RzOIP4ftq3MbZYsfH8kqSMBc2l1Ym2u3Dksv2/nR0zGQeNIgOxeMuwHU3Ry7+DdV1I96blPylVCCn/f5RAy6smKoaeylptPdXgVXmw3YBBUVYpHpm+xCIUUp9foAdb8hftO2DqUoHWdV/ZXt59zTAcsNFdQdB9A4525u+o4JlqK",
+        "sender_claimed_keys": {
+            "ed25519": "OsZMdC1gQ5nPr+L9tuT6xXsaFJkVPkgxP2FexHF1/QM"
+        },
+        "forwarding_curve25519_key_chain": []
+    }
+];
+
+/** An exported megolm session */
+export const BOB_MEGOLM_SESSION_DATA: IMegolmSessionData = {
+    "algorithm": "m.megolm.v1.aes-sha2",
+    "room_id": "!room:id",
+    "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+    "session_id": "gywydBrIJcJWktC/ic3tunKZM1XZm1MpYiYtdbj8Rpc",
+    "session_key": "AQAAAADZJL7OdM/KHfPzXPZ3CtlLBIlzbwk06dnZTd3bvkcdP5u73rdmThBKdqGA4xzCyxZsHdYLZRrlmD3VwOmNfvWMqYdPxA1X0vs3d172y9EIG8i+N/skJxTRypcVSV9XoinBNIWr/gkyepuAKiQqemlc8J5amD9OkmbVkmnrxP1uyYMsMnQayCXCVpLQv4nN7bpymTNV2ZtTKWImLXW4/EaX",
+    "sender_claimed_keys": {
+        "ed25519": "zBdpQwWYyz1MkZuEUhXqcdMfUNN/B9psLFDDDTJOg64"
+    },
+    "forwarding_curve25519_key_chain": []
+};
+
+/** A ratcheted version of BOB_MEGOLM_SESSION_DATA */
+export const BOB_RATCHTED_MEGOLM_SESSION_DATA: IMegolmSessionData = {
+    "algorithm": "m.megolm.v1.aes-sha2",
+    "room_id": "!room:id",
+    "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+    "session_id": "gywydBrIJcJWktC/ic3tunKZM1XZm1MpYiYtdbj8Rpc",
+    "session_key": "AQAAAAHZJL7OdM/KHfPzXPZ3CtlLBIlzbwk06dnZTd3bvkcdP5u73rdmThBKdqGA4xzCyxZsHdYLZRrlmD3VwOmNfvWMqYdPxA1X0vs3d172y9EIG8i+N/skJxTRypcVSV9Xoil2JdGx9oPqR0dFVh661Aqs86rJRbQ4IeRiuEm35VMxboMsMnQayCXCVpLQv4nN7bpymTNV2ZtTKWImLXW4/EaX",
+    "sender_claimed_keys": {
+        "ed25519": "zBdpQwWYyz1MkZuEUhXqcdMfUNN/B9psLFDDDTJOg64"
+    },
+    "forwarding_curve25519_key_chain": []
+};
+
+/** The key from BOB_MEGOLM_SESSION_DATA, encrypted for backup using `m.megolm_backup.v1.curve25519-aes-sha2` algorithm*/
+export const BOB_CURVE25519_KEY_BACKUP_DATA: KeyBackupSession = {
+    "first_message_index": 1,
+    "forwarded_count": 0,
+    "is_verified": false,
+    "session_data": {
+        "ciphertext": "d7UVOK17WEVky/8hK0h3HsTQrFMEbKbfqMcl2KtyTWcI9S5gGFWK9Git5BzVRxRggvxQ0c8PDfqL+dr3zHytAMW+71BJqIPQW910vV7SX3IcGylnoUcS3doVkJZiprXytXMP89AKcgv5Dj7mS2ZdvNGE+Atro74bzZ5yot5BrE0ZE5SjoUBPLaLMMu9HopLIV+qx01Rc3F0wmkocSPo51N0nv6wvO5Cst0FiOGHDK6r1pFlgDEJLmBkOyC4e8oMVbKTJzsSQVbJ8tJ37xuhI+T5P0ZlmiqKDqYRp8uh50w+txLEixYhEUunFgCTt1DAmiS9pLNYhLyl1ggwuQjzZe+AV6timbRxNJy18/AEcPomJw7z/pxYIiNLHRKOC13Wp8kGWx9cOgfMQ5KmBuLS8psGiLTBkfWPLOfNYqjbeqAR+OGZQoS6hUjbBYU7QuFa4FOYBHkNB2UqNsdsMb9qB/qs7QGTSb8Lok5YjW1c81BUpmIyKvuqnKma0MZskrpTYGQD2eJDABFCZwLFm+LgDyUTeSiV5xguYztLrHOk8LHKo9M8dIZgoBjeFVJxyjbcXKsVS3aQkMXKCrRlKLqhZTws/ZJwVfW9DbktZ9dT+tRZQvI7tjJofojcLX61AGJDnqUf5+2Gv1tEnmUI953gIzc8NlcFabPOsDsZEODt7MdOCTPT3w29umyhKbCsslpb64LoS/AB2QRPRCgkJS7snRA",
+        "ephemeral": "oO0VX84OUIzm2i/12zAhTWOZT5IFRH5mXaKZ8fXkCgU",
+        "mac": "lEfHlqfJQwU"
+    }
+};
+
+/** A test clear event */
+export const BOB_CLEAR_EVENT: Partial<IEvent> = {
+    "type": "m.room.message",
+    "room_id": "!room:id",
+    "sender": "@alice:localhost",
+    "content": {
+        "msgtype": "m.text",
+        "body": "Hello world"
+    }
+};
+
+/** The encrypted CLEAR_EVENT by MEGOLM_SESSION_DATA */
+export const BOB_ENCRYPTED_EVENT: Partial<IEvent> = {
+    "type": "m.room.encrypted",
+    "room_id": "!room:id",
+    "sender": "@alice:localhost",
+    "content": {
+        "algorithm": "m.megolm.v1.aes-sha2",
+        "sender_key": "FOvlmz18LLI3k/llCpqRoKT90+gFF8YhuL+v1YBXHlw",
+        "ciphertext": "AwgAEnA/mEqZm2lSrhoG11OpDqsohGSBJWsudbuoItLlivmpFZQHrKMbE6z/dhCTwUi76vwfRCtf4tyPMD845cqZH1nL0bowq3/awyzZ8Q263Y3WrLfkUTFBU6oPF/IULUFZZuw6kLdfd5g5+uigvqUhFFpICoj7KNHznv4sFNssd00/WgJquZ6PRt6e1v6ANFNiZPAwghIL+kBc6pb8i6MUWt9JnXilJhTqFDHdXiY4qkaKBWbwebC26PYM",
+        "session_id": "gywydBrIJcJWktC/ic3tunKZM1XZm1MpYiYtdbj8Rpc",
+        "device_id": "TEST_DEVICE"
+    },
+    "event_id": "$event1",
+    "origin_server_ts": 1507753886000
+};
+

spec/test-utils/test-utils.ts

@@ -6,9 +6,19 @@ import "../olm-loader";
 
 import { logger } from "../../src/logger";
 import { IContent, IEvent, IEventRelation, IUnsigned, MatrixEvent, MatrixEventEvent } from "../../src/models/event";
-import { ClientEvent, EventType, IPusher, MatrixClient, MsgType, RelationType } from "../../src";
+import {
+    ClientEvent,
+    EventType,
+    IJoinedRoom,
+    IPusher,
+    ISyncResponse,
+    MatrixClient,
+    MsgType,
+    RelationType,
+} from "../../src";
 import { SyncState } from "../../src/sync";
 import { eventMapperFor } from "../../src/event-mapper";
+import { TEST_ROOM_ID } from "./test-data";
 
 /**
  * Return a promise that is resolved when the client next emits a
@@ -39,6 +49,62 @@ export function syncPromise(client: MatrixClient, count = 1): Promise<void> {
     });
 }
 
+/**
+ * Return a sync response which contains a single room (by default TEST_ROOM_ID), with the members given
+ * @param roomMembers
+ * @param roomId
+ *
+ * @returns the sync response
+ */
+export function getSyncResponse(roomMembers: string[], roomId = TEST_ROOM_ID): ISyncResponse {
+    const roomResponse: IJoinedRoom = {
+        summary: {
+            "m.heroes": [],
+            "m.joined_member_count": roomMembers.length,
+            "m.invited_member_count": roomMembers.length,
+        },
+        state: {
+            events: [
+                mkEventCustom({
+                    sender: roomMembers[0],
+                    type: "m.room.encryption",
+                    state_key: "",
+                    content: {
+                        algorithm: "m.megolm.v1.aes-sha2",
+                    },
+                }),
+            ],
+        },
+        timeline: {
+            events: [],
+            prev_batch: "",
+        },
+        ephemeral: { events: [] },
+        account_data: { events: [] },
+        unread_notifications: {},
+    };
+
+    for (let i = 0; i < roomMembers.length; i++) {
+        roomResponse.state.events.push(
+            mkMembershipCustom({
+                membership: "join",
+                sender: roomMembers[i],
+            }),
+        );
+    }
+
+    return {
+        next_batch: "1",
+        rooms: {
+            join: { [roomId]: roomResponse },
+            invite: {},
+            leave: {},
+            knock: {},
+        },
+        account_data: { events: [] },
+    };
+}
+
 /**
  * Create a spy for an object and automatically spy its methods.
  * @param constr - The class constructor (used with 'new')
@@ -249,6 +315,7 @@ export interface IMessageOpts {
     event?: boolean;
     relatesTo?: IEventRelation;
     ts?: number;
+    unsigned?: IUnsigned;
 }
 
 /**
@@ -455,15 +522,22 @@ export async function awaitDecryption(
     }
 
     return new Promise((resolve) => {
-        event.once(MatrixEventEvent.Decrypted, (ev, err) => {
-            logger.log(`${Date.now()}: MatrixEventEvent.Decrypted for event ${event.getId()}: ${err ?? "success"}`);
-            resolve(ev);
-        });
+        if (waitOnDecryptionFailure) {
+            event.on(MatrixEventEvent.Decrypted, (ev, err) => {
+                logger.log(`${Date.now()}: MatrixEventEvent.Decrypted for event ${event.getId()}: ${err ?? "success"}`);
+                if (!err) {
+                    resolve(ev);
+                }
+            });
+        } else {
+            event.once(MatrixEventEvent.Decrypted, (ev, err) => {
+                logger.log(`${Date.now()}: MatrixEventEvent.Decrypted for event ${event.getId()}: ${err ?? "success"}`);
+                resolve(ev);
+            });
+        }
     });
 }
 
-export const emitPromise = (e: EventEmitter, k: string): Promise<any> => new Promise((r) => e.once(k, r));
-
 export const mkPusher = (extra: Partial<IPusher> = {}): IPusher => ({
     app_display_name: "app",
     app_id: "123",
@@ -486,3 +560,25 @@ CRYPTO_BACKENDS["rust-sdk"] = (client: MatrixClient) => client.initRustCrypto();
 if (global.Olm) {
     CRYPTO_BACKENDS["libolm"] = (client: MatrixClient) => client.initCrypto();
 }
+
+export const emitPromise = (e: EventEmitter, k: string): Promise<any> => new Promise((r) => e.once(k, r));
+
+/**
+ * Advance the fake timers in a loop until the given promise resolves or rejects.
+ *
+ * Returns the result of the promise.
+ *
+ * This can be useful when there are multiple steps in the code which require an iteration of the event loop.
+ */
+export async function advanceTimersUntil<T>(promise: Promise<T>): Promise<T> {
+    let resolved = false;
+    promise.finally(() => {
+        resolved = true;
+    });
+
+    while (!resolved) {
+        await jest.advanceTimersByTimeAsync(1);
+    }
+
+    return await promise;
+}

spec/test-utils/thread.ts

@@ -18,7 +18,7 @@ import { RelationType } from "../../src/@types/event";
 import { MatrixClient } from "../../src/client";
 import { MatrixEvent, MatrixEventEvent } from "../../src/models/event";
 import { Room } from "../../src/models/room";
-import { Thread } from "../../src/models/thread";
+import { Thread, THREAD_RELATION_TYPE } from "../../src/models/thread";
 import { mkMessage } from "./test-utils";
 
 export const makeThreadEvent = ({
@@ -34,7 +34,7 @@ export const makeThreadEvent = ({
         ...props,
         relatesTo: {
             event_id: rootEventId,
-            rel_type: "m.thread",
+            rel_type: THREAD_RELATION_TYPE.name,
             ["m.in_reply_to"]: {
                 event_id: replyToEventId,
             },
@@ -157,7 +157,27 @@ export const mkThread = ({
         room?.reEmitter.reEmit(evt, [MatrixEventEvent.BeforeRedaction]);
     }
 
-    const thread = room.createThread(rootEvent.getId() ?? "", rootEvent, events, true);
+    const thread = room.createThread(rootEvent.getId() ?? "", rootEvent, [rootEvent, ...events], true);
 
     return { thread, rootEvent, events };
 };
+
+/**
+ * Create a thread, and make sure the events are added to the thread and the
+ * room's timeline as if they came in via sync.
+ *
+ * Note that mkThread doesn't actually add the events properly to the room.
+ */
+export const populateThread = ({
+    room,
+    client,
+    authorId,
+    participantUserIds,
+    length = 2,
+    ts = 1,
+}: MakeThreadProps): MakeThreadResult => {
+    const ret = mkThread({ room, client, authorId, participantUserIds, length, ts });
+    ret.thread.initialEventsFetched = true;
+    room.addLiveEvents(ret.events);
+    return ret;
+};

spec/test-utils/webrtc.ts

@@ -485,6 +485,7 @@ export class MockCallMatrixClient extends TypedEventEmitter<EmittedEvents, Emitt
 
     public getRooms = jest.fn<Room[], []>().mockReturnValue([]);
     public getRoom = jest.fn();
+    public getFoci = jest.fn();
 
     public supportsThreads(): boolean {
         return true;

spec/unit/autodiscovery.spec.ts

@@ -15,10 +15,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import fetchMock from "fetch-mock-jest";
 import MockHttpBackend from "matrix-mock-request";
 
+import { AutoDiscoveryAction, M_AUTHENTICATION } from "../../src";
 import { AutoDiscovery } from "../../src/autodiscovery";
-import { OidcDiscoveryError } from "../../src/oidc/validate";
+import { OidcError } from "../../src/oidc/error";
+import { makeDelegatedAuthConfig } from "../test-utils/oidc";
+
+// keep to reset the fetch function after using MockHttpBackend
+// @ts-ignore private property
+const realAutoDiscoveryFetch: typeof global.fetch = AutoDiscovery.fetchFn;
 
 describe("AutoDiscovery", function () {
     const getHttpBackend = (): MockHttpBackend => {
@@ -27,6 +34,10 @@ describe("AutoDiscovery", function () {
         return httpBackend;
     };
 
+    afterAll(() => {
+        AutoDiscovery.setFetchFn(realAutoDiscoveryFetch);
+    });
+
     it("should throw an error when no domain is specified", function () {
         getHttpBackend();
         return Promise.all([
@@ -340,7 +351,7 @@ describe("AutoDiscovery", function () {
         function () {
             const httpBackend = getHttpBackend();
             httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
-                not_matrix_versions: ["r0.0.1"],
+                not_matrix_versions: ["v1.1"],
             });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -377,7 +388,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["r0.0.1"],
+                versions: ["v1.1"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -400,7 +411,7 @@ describe("AutoDiscovery", function () {
                     },
                     "m.authentication": {
                         state: "IGNORE",
-                        error: OidcDiscoveryError.NotSupported,
+                        error: OidcError.NotSupported,
                     },
                 };
 
@@ -417,7 +428,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["r0.0.1"],
+                versions: ["v1.1"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -441,7 +452,7 @@ describe("AutoDiscovery", function () {
                     },
                     "m.authentication": {
                         state: "IGNORE",
-                        error: OidcDiscoveryError.NotSupported,
+                        error: OidcError.NotSupported,
                     },
                 };
 
@@ -458,7 +469,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["r0.0.1"],
+                versions: ["v1.1"],
             });
         httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
             "m.homeserver": {
@@ -485,7 +496,7 @@ describe("AutoDiscovery", function () {
                     },
                     "m.authentication": {
                         state: "FAIL_ERROR",
-                        error: OidcDiscoveryError.Misconfigured,
+                        error: OidcError.Misconfigured,
                     },
                 };
 
@@ -504,7 +515,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["r0.0.1"],
+                    versions: ["v1.1"],
                 });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -549,7 +560,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["r0.0.1"],
+                    versions: ["v1.1"],
                 });
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
                 "m.homeserver": {
@@ -595,7 +606,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["r0.0.1"],
+                    versions: ["v1.1"],
                 });
             httpBackend.when("GET", "/_matrix/identity/v2").respond(404, {});
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
@@ -642,7 +653,7 @@ describe("AutoDiscovery", function () {
                     expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
                 })
                 .respond(200, {
-                    versions: ["r0.0.1"],
+                    versions: ["v1.1"],
                 });
             httpBackend.when("GET", "/_matrix/identity/v2").respond(500, {});
             httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
@@ -686,7 +697,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["r0.0.1"],
+                versions: ["v1.1"],
             });
         httpBackend
             .when("GET", "/_matrix/identity/v2")
@@ -719,7 +730,7 @@ describe("AutoDiscovery", function () {
                     },
                     "m.authentication": {
                         state: "IGNORE",
-                        error: OidcDiscoveryError.NotSupported,
+                        error: OidcError.NotSupported,
                     },
                 };
 
@@ -736,7 +747,7 @@ describe("AutoDiscovery", function () {
                 expect(req.path).toEqual("https://chat.example.org/_matrix/client/versions");
             })
             .respond(200, {
-                versions: ["r0.0.1"],
+                versions: ["v1.1"],
             });
         httpBackend
             .when("GET", "/_matrix/identity/v2")
@@ -775,7 +786,7 @@ describe("AutoDiscovery", function () {
                     },
                     "m.authentication": {
                         state: "IGNORE",
-                        error: OidcDiscoveryError.NotSupported,
+                        error: OidcError.NotSupported,
                     },
                 };
 
@@ -855,4 +866,106 @@ describe("AutoDiscovery", function () {
             }),
         ]);
     });
+
+    it("should FAIL_ERROR for unsupported Matrix version", () => {
+        const httpBackend = getHttpBackend();
+        httpBackend.when("GET", "/.well-known/matrix/client").respond(200, {
+            "m.homeserver": {
+                base_url: "https://example.org",
+            },
+        });
+        httpBackend.when("GET", "/_matrix/client/versions").respond(200, {
+            versions: ["r0.6.0"],
+        });
+        return Promise.all([
+            httpBackend.flushAllExpected(),
+            AutoDiscovery.findClientConfig("example.org").then((conf) => {
+                const expected = {
+                    "m.homeserver": {
+                        state: AutoDiscoveryAction.FAIL_ERROR,
+                        error: AutoDiscovery.ERROR_HOMESERVER_TOO_OLD,
+                        base_url: "https://example.org",
+                    },
+                    "m.identity_server": {
+                        state: "PROMPT",
+                        error: null,
+                        base_url: null,
+                    },
+                };
+
+                expect(conf).toEqual(expected);
+            }),
+        ]);
+    });
+
+    describe("m.authentication", () => {
+        const homeserverName = "example.org";
+        const homeserverUrl = "https://chat.example.org/";
+        const issuer = "https://auth.org/";
+
+        beforeAll(() => {
+            // make these tests independent from fetch mocking above
+            AutoDiscovery.setFetchFn(realAutoDiscoveryFetch);
+        });
+
+        beforeEach(() => {
+            fetchMock.resetBehavior();
+            fetchMock.get(`${homeserverUrl}_matrix/client/versions`, { versions: ["v1.1"] });
+
+            fetchMock.get("https://example.org/.well-known/matrix/client", {
+                "m.homeserver": {
+                    // Note: we also expect this test to trim the trailing slash
+                    base_url: "https://chat.example.org/",
+                },
+                "m.authentication": {
+                    issuer,
+                },
+            });
+        });
+
+        it("should return valid authentication configuration", async () => {
+            const config = makeDelegatedAuthConfig(issuer);
+
+            fetchMock.get(`${config.metadata.issuer}.well-known/openid-configuration`, config.metadata);
+            fetchMock.get(`${config.metadata.issuer}jwks`, {
+                status: 200,
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                keys: [],
+            });
+
+            const result = await AutoDiscovery.findClientConfig(homeserverName);
+
+            expect(result[M_AUTHENTICATION.stable!]).toEqual({
+                state: AutoDiscovery.SUCCESS,
+                ...config,
+                signingKeys: [],
+                account: undefined,
+                error: null,
+            });
+        });
+
+        it("should set state to error for invalid authentication configuration", async () => {
+            const config = makeDelegatedAuthConfig(issuer);
+            // authorization_code is required
+            config.metadata.grant_types_supported = ["openid"];
+
+            fetchMock.get(`${config.metadata.issuer}.well-known/openid-configuration`, config.metadata);
+            fetchMock.get(`${config.metadata.issuer}jwks`, {
+                status: 200,
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                keys: [],
+            });
+
+            const result = await AutoDiscovery.findClientConfig(homeserverName);
+
+            expect(result[M_AUTHENTICATION.stable!]).toEqual({
+                state: AutoDiscovery.FAIL_ERROR,
+                error: OidcError.OpSupport,
+            });
+        });
+    });
 });

spec/unit/base64.spec.ts

@@ -0,0 +1,88 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { TextEncoder, TextDecoder } from "util";
+import NodeBuffer from "node:buffer";
+
+import { decodeBase64, encodeBase64, encodeUnpaddedBase64, encodeUnpaddedBase64Url } from "../../src/base64";
+
+describe.each(["browser", "node"])("Base64 encoding (%s)", (env) => {
+    let origBuffer = Buffer;
+
+    beforeAll(() => {
+        if (env === "browser") {
+            origBuffer = Buffer;
+            // @ts-ignore
+            // eslint-disable-next-line no-global-assign
+            Buffer = undefined;
+
+            global.atob = NodeBuffer.atob;
+            global.btoa = NodeBuffer.btoa;
+        }
+    });
+
+    afterAll(() => {
+        // eslint-disable-next-line no-global-assign
+        Buffer = origBuffer;
+        // @ts-ignore
+        global.atob = undefined;
+        // @ts-ignore
+        global.btoa = undefined;
+    });
+
+    it("Should decode properly encoded data", () => {
+        const decoded = new TextDecoder().decode(decodeBase64("ZW5jb2RpbmcgaGVsbG8gd29ybGQ="));
+
+        expect(decoded).toStrictEqual("encoding hello world");
+    });
+
+    it("Should encode unpadded URL-safe base64", () => {
+        const toEncode = "?????";
+        const data = new TextEncoder().encode(toEncode);
+
+        const encoded = encodeUnpaddedBase64Url(data);
+        expect(encoded).toEqual("Pz8_Pz8");
+    });
+
+    it("Should decode URL-safe base64", () => {
+        const decoded = new TextDecoder().decode(decodeBase64("Pz8_Pz8="));
+
+        expect(decoded).toStrictEqual("?????");
+    });
+
+    it("Encode unpadded should not have padding", () => {
+        const toEncode = "encoding hello world";
+        const data = new TextEncoder().encode(toEncode);
+
+        const paddedEncoded = encodeBase64(data);
+        const unpaddedEncoded = encodeUnpaddedBase64(data);
+
+        expect(paddedEncoded).not.toEqual(unpaddedEncoded);
+
+        const padding = paddedEncoded.charAt(paddedEncoded.length - 1);
+        expect(padding).toStrictEqual("=");
+    });
+
+    it("Decode should be indifferent to padding", () => {
+        const withPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ=";
+        const withoutPadding = "ZW5jb2RpbmcgaGVsbG8gd29ybGQ";
+
+        const decodedPad = decodeBase64(withPadding);
+        const decodedNoPad = decodeBase64(withoutPadding);
+
+        expect(decodedPad).toStrictEqual(decodedNoPad);
+    });
+});

spec/unit/content-repo.spec.ts

@@ -33,7 +33,7 @@ describe("ContentRepo", function () {
         it("should return a download URL if no width/height/resize are specified", function () {
             const mxcUri = "mxc://server.name/resourceid";
             expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
-                baseUrl + "/_matrix/media/r0/download/server.name/resourceid",
+                baseUrl + "/_matrix/media/v3/download/server.name/resourceid",
             );
         });
 
@@ -44,21 +44,21 @@ describe("ContentRepo", function () {
         it("should return a thumbnail URL if a width/height/resize is specified", function () {
             const mxcUri = "mxc://server.name/resourceid";
             expect(getHttpUriForMxc(baseUrl, mxcUri, 32, 64, "crop")).toEqual(
-                baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" + "?width=32&height=64&method=crop",
+                baseUrl + "/_matrix/media/v3/thumbnail/server.name/resourceid" + "?width=32&height=64&method=crop",
             );
         });
 
         it("should put fragments from mxc:// URIs after any query parameters", function () {
             const mxcUri = "mxc://server.name/resourceid#automade";
             expect(getHttpUriForMxc(baseUrl, mxcUri, 32)).toEqual(
-                baseUrl + "/_matrix/media/r0/thumbnail/server.name/resourceid" + "?width=32#automade",
+                baseUrl + "/_matrix/media/v3/thumbnail/server.name/resourceid" + "?width=32#automade",
             );
         });
 
         it("should put fragments from mxc:// URIs at the end of the HTTP URI", function () {
             const mxcUri = "mxc://server.name/resourceid#automade";
             expect(getHttpUriForMxc(baseUrl, mxcUri)).toEqual(
-                baseUrl + "/_matrix/media/r0/download/server.name/resourceid#automade",
+                baseUrl + "/_matrix/media/v3/download/server.name/resourceid#automade",
             );
         });
     });

spec/unit/crypto.spec.ts

@@ -15,11 +15,16 @@ import { sleep } from "../../src/utils";
 import { CRYPTO_ENABLED } from "../../src/client";
 import { DeviceInfo } from "../../src/crypto/deviceinfo";
 import { logger } from "../../src/logger";
-import { MemoryStore } from "../../src";
+import { DeviceVerification, MemoryStore } from "../../src";
 import { RoomKeyRequestState } from "../../src/crypto/OutgoingRoomKeyRequestManager";
 import { RoomMember } from "../../src/models/room-member";
 import { IStore } from "../../src/store";
 import { IRoomEncryption, RoomList } from "../../src/crypto/RoomList";
+import { EventShieldColour, EventShieldReason } from "../../src/crypto-api";
+import { UserTrustLevel } from "../../src/crypto/CrossSigning";
+import { CryptoBackend } from "../../src/common-crypto/CryptoBackend";
+import { EventDecryptionResult } from "../../src/common-crypto/CryptoBackend";
+import * as testData from "../test-utils/test-data";
 
 const Olm = global.Olm;
 
@@ -110,14 +115,25 @@ describe("Crypto", function () {
         expect(Crypto.getOlmVersion()[0]).toEqual(3);
     });
 
+    it("getVersion() should return the current version of the olm library", async () => {
+        const client = new TestClient("@alice:example.com", "deviceid").client;
+        await client.initCrypto();
+
+        const olmVersionTuple = Crypto.getOlmVersion();
+        expect(client.getCrypto()?.getVersion()).toBe(
+            `Olm ${olmVersionTuple[0]}.${olmVersionTuple[1]}.${olmVersionTuple[2]}`,
+        );
+    });
+
     describe("encrypted events", function () {
-        it("provides encryption information", async function () {
+        it("provides encryption information for events from unverified senders", async function () {
             const client = new TestClient("@alice:example.com", "deviceid").client;
             await client.initCrypto();
 
             // unencrypted event
             const event = {
                 getId: () => "$event_id",
+                getSender: () => "@bob:example.com",
                 getSenderKey: () => null,
                 getWireContent: () => {
                     return {};
@@ -127,6 +143,8 @@ describe("Crypto", function () {
             let encryptionInfo = client.getEventEncryptionInfo(event);
             expect(encryptionInfo.encrypted).toBeFalsy();
 
+            expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toBe(null);
+
             // unknown sender (e.g. deleted device), forwarded megolm key (untrusted)
             event.getSenderKey = () => "YmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmJiYmI";
             event.getWireContent = () => {
@@ -141,6 +159,11 @@ describe("Crypto", function () {
             expect(encryptionInfo.authenticated).toBeFalsy();
             expect(encryptionInfo.sender).toBeFalsy();
 
+            expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                shieldColour: EventShieldColour.GREY,
+                shieldReason: EventShieldReason.AUTHENTICITY_NOT_GUARANTEED,
+            });
+
             // known sender, megolm key from backup
             event.getForwardingCurve25519KeyChain = () => [];
             event.isKeySourceUntrusted = () => true;
@@ -155,6 +178,11 @@ describe("Crypto", function () {
             expect(encryptionInfo.sender).toBeTruthy();
             expect(encryptionInfo.mismatchedSender).toBeFalsy();
 
+            expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                shieldColour: EventShieldColour.GREY,
+                shieldReason: EventShieldReason.AUTHENTICITY_NOT_GUARANTEED,
+            });
+
             // known sender, trusted megolm key, but bad ed25519key
             event.isKeySourceUntrusted = () => false;
             device.keys["ed25519:FLIBBLE"] = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB";
@@ -165,9 +193,115 @@ describe("Crypto", function () {
             expect(encryptionInfo.sender).toBeTruthy();
             expect(encryptionInfo.mismatchedSender).toBeTruthy();
 
+            expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                shieldColour: EventShieldColour.RED,
+                shieldReason: EventShieldReason.MISMATCHED_SENDER_KEY,
+            });
+
             client.stopClient();
         });
 
+        describe("provides encryption information for events from verified senders", function () {
+            const testDeviceId = testData.BOB_TEST_DEVICE_ID;
+            const testDevice = testData.BOB_SIGNED_TEST_DEVICE_DATA;
+
+            let client: MatrixClient;
+            beforeEach(async () => {
+                client = new TestClient("@alice:example.com", "deviceid").client;
+                await client.initCrypto();
+
+                // mock out the verification check
+                client.crypto!.checkUserTrust = (userId) => new UserTrustLevel(true, false, false);
+            });
+
+            afterEach(() => {
+                client.stopClient();
+            });
+
+            async function buildEncryptedEvent(
+                decryptionResult: Partial<EventDecryptionResult> = {},
+            ): Promise<MatrixEvent> {
+                const mockCryptoBackend = {
+                    decryptEvent: async (event: MatrixEvent): Promise<EventDecryptionResult> => {
+                        return {
+                            claimedEd25519Key: testDevice.keys["ed25519:" + testDeviceId],
+                            clearEvent: {
+                                room_id: "!room_id",
+                                type: "m.room.message",
+                                content: { body: "test" },
+                            },
+                            forwardingCurve25519KeyChain: [],
+                            senderCurve25519Key: testDevice.keys["curve25519:" + testDeviceId],
+                            ...decryptionResult,
+                        };
+                    },
+                } as unknown as CryptoBackend;
+
+                const event = new MatrixEvent({
+                    event_id: "$event_id",
+                    sender: testData.BOB_TEST_USER_ID,
+                    type: "m.room.encrypted",
+                    content: { algorithm: "m.megolm.v1.aes-sha2" },
+                });
+                await event.attemptDecryption(mockCryptoBackend);
+                return event;
+            }
+
+            it("unknown device", async () => {
+                const event = await buildEncryptedEvent();
+                expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                    shieldColour: EventShieldColour.GREY,
+                    shieldReason: EventShieldReason.UNKNOWN_DEVICE,
+                });
+            });
+
+            it("known but unsigned device", async () => {
+                client.crypto!.deviceList.storeDevicesForUser(testData.BOB_TEST_USER_ID, {
+                    [testDeviceId]: {
+                        keys: testDevice.keys,
+                        algorithms: testDevice.algorithms,
+                        verified: DeviceVerification.Unverified,
+                        known: true,
+                    },
+                });
+
+                const event = await buildEncryptedEvent();
+                expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                    shieldColour: EventShieldColour.RED,
+                    shieldReason: EventShieldReason.UNSIGNED_DEVICE,
+                });
+            });
+
+            describe("known and verified device", () => {
+                beforeEach(() => {
+                    client.crypto!.deviceList.storeDevicesForUser(testData.BOB_TEST_USER_ID, {
+                        [testDeviceId]: {
+                            keys: testDevice.keys,
+                            algorithms: testDevice.algorithms,
+                            verified: DeviceVerification.Verified,
+                            known: true,
+                        },
+                    });
+                });
+
+                it("regular key", async () => {
+                    const event = await buildEncryptedEvent();
+                    expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                        shieldColour: EventShieldColour.NONE,
+                        shieldReason: null,
+                    });
+                });
+
+                it("unauthenticated key", async () => {
+                    const event = await buildEncryptedEvent({ untrusted: true });
+                    expect(await client.getCrypto()!.getEncryptionInfoForEvent(event)).toEqual({
+                        shieldColour: EventShieldColour.GREY,
+                        shieldReason: EventShieldReason.AUTHENTICITY_NOT_GUARANTEED,
+                    });
+                });
+            });
+        });
+
         it("doesn't throw an error when attempting to decrypt a redacted event", async () => {
             const client = new TestClient("@alice:example.com", "deviceid").client;
             await client.initCrypto();
@@ -222,7 +356,6 @@ describe("Crypto", function () {
 
         let crypto: Crypto;
         let mockBaseApis: MatrixClient;
-        let mockRoomList: RoomList;
 
         let fakeEmitter: EventEmitter;
 
@@ -256,19 +389,10 @@ describe("Crypto", function () {
                 isGuest: jest.fn(),
                 emit: jest.fn(),
             } as unknown as MatrixClient;
-            mockRoomList = {} as unknown as RoomList;
 
             fakeEmitter = new EventEmitter();
 
-            crypto = new Crypto(
-                mockBaseApis,
-                "@alice:home.server",
-                "FLIBBLE",
-                clientStore,
-                cryptoStore,
-                mockRoomList,
-                [],
-            );
+            crypto = new Crypto(mockBaseApis, "@alice:home.server", "FLIBBLE", clientStore, cryptoStore, []);
             crypto.registerEventHandlers(fakeEmitter as any);
             await crypto.init();
         });
@@ -982,7 +1106,7 @@ describe("Crypto", function () {
 
     describe("Secret storage", function () {
         it("creates secret storage even if there is no keyInfo", async function () {
-            jest.spyOn(logger, "log").mockImplementation(() => {});
+            jest.spyOn(logger, "debug").mockImplementation(() => {});
             jest.setTimeout(10000);
             const client = new TestClient("@a:example.com", "dev").client;
             await client.initCrypto();
@@ -1207,15 +1331,9 @@ describe("Crypto", function () {
                 setRoomEncryption: jest.fn().mockResolvedValue(undefined),
             } as unknown as RoomList;
 
-            crypto = new Crypto(
-                mockClient,
-                "@alice:home.server",
-                "FLIBBLE",
-                clientStore,
-                cryptoStore,
-                mockRoomList,
-                [],
-            );
+            crypto = new Crypto(mockClient, "@alice:home.server", "FLIBBLE", clientStore, cryptoStore, []);
+            // @ts-ignore we are injecting a mock into a private property
+            crypto.roomList = mockRoomList;
         });
 
         it("should set the algorithm if called for a known room", async () => {

spec/unit/crypto/DeviceList.spec.ts

@@ -129,7 +129,7 @@ describe("DeviceList", function () {
         });
     });
 
-    it("should have an outdated devicelist on an invalidation while an " + "update is in progress", function () {
+    it("should have an outdated devicelist on an invalidation while an update is in progress", async function () {
         const dl = createTestDeviceList();
 
         dl.startTrackingDeviceList("@test1:sw1v.org");
@@ -148,11 +148,8 @@ describe("DeviceList", function () {
         dl.invalidateUserDeviceList("@test1:sw1v.org");
         dl.refreshOutdatedDeviceLists();
 
-        // TODO: Fix this test so we actually await the call and assertions and remove
-        // the eslint disable, https://github.com/matrix-org/matrix-js-sdk/issues/2977
-        //
-        // eslint-disable-next-line jest/valid-expect-in-promise
-        dl.saveIfDirty()
+        await dl
+            .saveIfDirty()
             .then(() => {
                 // the first request completes
                 queryDefer1.resolve({
@@ -163,12 +160,13 @@ describe("DeviceList", function () {
                 });
                 return prom1;
             })
-            .then(() => {
+            .then(async () => {
                 // uh-oh; user restarts before second request completes. The new instance
                 // should know we never got a complete device list.
                 logger.log("Creating new devicelist to simulate app reload");
                 downloadSpy.mockReset();
                 const dl2 = createTestDeviceList();
+                await dl2.load();
                 const queryDefer3 = utils.defer<IDownloadKeyResult>();
                 downloadSpy.mockReturnValue(queryDefer3.promise);
 

spec/unit/crypto/backup.spec.ts

@@ -215,6 +215,36 @@ describe("MegolmBackup", function () {
             jest.spyOn(global, "setTimeout").mockRestore();
         });
 
+        test("fail if crypto not enabled", async () => {
+            const client = makeTestClient(cryptoStore);
+            const data = {
+                algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
+                version: "1",
+                auth_data: {
+                    public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                },
+            };
+            await expect(client.restoreKeyBackupWithSecretStorage(data)).rejects.toThrow(
+                "End-to-end encryption disabled",
+            );
+        });
+
+        test("fail if given backup has no version", async () => {
+            const client = makeTestClient(cryptoStore);
+            await client.initCrypto();
+            const data = {
+                algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM,
+                auth_data: {
+                    public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo",
+                },
+            };
+            const key = Uint8Array.from([1, 2, 3, 4, 5, 6, 7, 8]);
+            await client.getCrypto()!.storeSessionBackupPrivateKey(key, "1");
+            await expect(client.restoreKeyBackupWithCache(undefined, undefined, data)).rejects.toThrow(
+                "Backup version must be defined",
+            );
+        });
+
         it("automatically calls the key back up", function () {
             const groupSession = new Olm.OutboundGroupSession();
             groupSession.create();

spec/unit/crypto/secrets.spec.ts

@@ -28,6 +28,7 @@ import { DeviceInfo } from "../../../src/crypto/deviceinfo";
 import { ISignatures } from "../../../src/@types/signed";
 import { ICurve25519AuthData } from "../../../src/crypto/keybackup";
 import { SecretStorageKeyDescription, SECRET_STORAGE_ALGORITHM_V1_AES } from "../../../src/secret-storage";
+import { decodeBase64 } from "../../../src/base64";
 
 async function makeTestClient(
     userInfo: { userId: string; deviceId: string },
@@ -275,13 +276,13 @@ describe("Secrets", function () {
 
     describe("bootstrap", function () {
         // keys used in some of the tests
-        const XSK = new Uint8Array(olmlib.decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="));
+        const XSK = new Uint8Array(decodeBase64("3lo2YdJugHjfE+Or7KJ47NuKbhE7AAGLgQ/dc19913Q="));
         const XSPubKey = "DRb8pFVJyEJ9OWvXeUoM0jq/C2Wt+NxzBZVuk2nRb+0";
-        const USK = new Uint8Array(olmlib.decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="));
+        const USK = new Uint8Array(decodeBase64("lKWi3hJGUie5xxHgySoz8PHFnZv6wvNaud/p2shN9VU="));
         const USPubKey = "CUpoiTtHiyXpUmd+3ohb7JVxAlUaOG1NYs9Jlx8soQU";
-        const SSK = new Uint8Array(olmlib.decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="));
+        const SSK = new Uint8Array(decodeBase64("1R6JVlXX99UcfUZzKuCDGQgJTw8ur1/ofgPD8pp+96M="));
         const SSPubKey = "0DfNsRDzEvkCLA0gD3m7VAGJ5VClhjEsewI35xq873Q";
-        const SSSSKey = new Uint8Array(olmlib.decodeBase64("XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0="));
+        const SSSSKey = new Uint8Array(decodeBase64("XrmITOOdBhw6yY5Bh7trb/bgp1FRdIGyCUxxMP873R0="));
 
         it("bootstraps when no storage or cross-signing keys locally", async function () {
             const key = new Uint8Array(16);
@@ -312,6 +313,7 @@ describe("Secrets", function () {
                 this.emit(ClientEvent.AccountData, event);
                 return {};
             };
+            bob.getKeyBackupVersion = jest.fn().mockResolvedValue(null);
 
             await bob.bootstrapCrossSigning({
                 authUploadDeviceSigningKeys: async (func) => {

spec/unit/crypto/verification/secret_request.spec.ts

@@ -16,7 +16,7 @@ limitations under the License.
 
 import "../../../olm-loader";
 import { MatrixClient, MatrixEvent } from "../../../../src/matrix";
-import { encodeBase64 } from "../../../../src/crypto/olmlib";
+import { encodeBase64 } from "../../../../src/base64";
 import "../../../../src/crypto"; // import this to cycle-break
 import { CrossSigningInfo } from "../../../../src/crypto/CrossSigning";
 import { VerificationRequest } from "../../../../src/crypto/verification/request/VerificationRequest";

spec/unit/embedded.spec.ts

@@ -23,7 +23,14 @@ limitations under the License.
 // eslint-disable-next-line no-restricted-imports
 import { EventEmitter } from "events";
 import { MockedObject } from "jest-mock";
-import { WidgetApi, WidgetApiToWidgetAction, MatrixCapabilities, ITurnServer, IRoomEvent } from "matrix-widget-api";
+import {
+    WidgetApi,
+    WidgetApiToWidgetAction,
+    MatrixCapabilities,
+    ITurnServer,
+    IRoomEvent,
+    IOpenIDCredentials,
+} from "matrix-widget-api";
 
 import { createRoomWidgetClient, MsgType } from "../../src/matrix";
 import { MatrixClient, ClientEvent, ITurnServer as IClientTurnServer } from "../../src/client";
@@ -33,6 +40,12 @@ import { MatrixEvent } from "../../src/models/event";
 import { ToDeviceBatch } from "../../src/models/ToDeviceMessage";
 import { DeviceInfo } from "../../src/crypto/deviceinfo";
 
+const testOIDCToken = {
+    access_token: "12345678",
+    expires_in: "10",
+    matrix_server_name: "homeserver.oabc",
+    token_type: "Bearer",
+};
 class MockWidgetApi extends EventEmitter {
     public start = jest.fn();
     public requestCapability = jest.fn();
@@ -49,8 +62,15 @@ class MockWidgetApi extends EventEmitter {
     public sendRoomEvent = jest.fn(() => ({ event_id: `$${Math.random()}` }));
     public sendStateEvent = jest.fn();
     public sendToDevice = jest.fn();
+    public requestOpenIDConnectToken = jest.fn(() => {
+        return testOIDCToken;
+        return new Promise<IOpenIDCredentials>(() => {
+            return testOIDCToken;
+        });
+    });
     public readStateEvents = jest.fn(() => []);
     public getTurnServers = jest.fn(() => []);
+    public sendContentLoaded = jest.fn();
 
     public transport = { reply: jest.fn() };
 }
@@ -285,7 +305,12 @@ describe("RoomWidgetClient", () => {
             expect(await emittedSync).toEqual(SyncState.Syncing);
         });
     });
-
+    describe("oidc token", () => {
+        it("requests an oidc token", async () => {
+            await makeClient({});
+            expect(await client.getOpenIdToken()).toStrictEqual(testOIDCToken);
+        });
+    });
     it("gets TURN servers", async () => {
         const server1: ITurnServer = {
             uris: [

spec/unit/event-mapper.spec.ts

@@ -34,6 +34,7 @@ describe("eventMapperFor", function () {
                 getRoom(roomId: string): Room | null {
                     return rooms.find((r) => r.roomId === roomId) ?? null;
                 },
+                setUserCreator(_) {},
             } as IStore,
             scheduler: {
                 setProcessFunction: jest.fn(),

spec/unit/event-timeline-set.spec.ts

@@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { mocked } from "jest-mock";
+
 import * as utils from "../test-utils/test-utils";
 import {
     DuplicateStrategy,
@@ -160,6 +162,33 @@ describe("EventTimelineSet", () => {
                 eventTimelineSet.addEventToTimeline(messageEvent, liveTimeline, true, false);
             }).not.toThrow();
         });
+
+        it("should aggregate relations which belong to unknown timeline without adding them to any timeline", () => {
+            // If threads are disabled all events go into the main timeline
+            mocked(client.supportsThreads).mockReturnValue(true);
+            const reactionEvent = utils.mkReaction(messageEvent, client, client.getSafeUserId(), roomId);
+
+            const liveTimeline = eventTimelineSet.getLiveTimeline();
+            expect(liveTimeline.getEvents().length).toStrictEqual(0);
+            eventTimelineSet.addEventToTimeline(reactionEvent, liveTimeline, {
+                toStartOfTimeline: true,
+            });
+            expect(liveTimeline.getEvents().length).toStrictEqual(0);
+
+            eventTimelineSet.addEventToTimeline(messageEvent, liveTimeline, {
+                toStartOfTimeline: true,
+            });
+            expect(liveTimeline.getEvents()).toHaveLength(1);
+            const [event] = liveTimeline.getEvents();
+            const reactions = eventTimelineSet.relations!.getChildEventsForEvent(
+                event.getId()!,
+                "m.annotation",
+                "m.reaction",
+            )!;
+            const relations = reactions.getRelations();
+            expect(relations).toHaveLength(1);
+            expect(relations[0].getId()).toBe(reactionEvent.getId());
+        });
     });
 
     describe("addEventToTimeline (thread timeline)", () => {

spec/unit/http-api/__snapshots__/index.spec.ts.snap

@@ -6,6 +6,6 @@ exports[`MatrixHttpApi should return expected object from \`getContentUri\` 1`]
   "params": {
     "access_token": "token",
   },
-  "path": "/_matrix/media/r0/upload",
+  "path": "/_matrix/media/v3/upload",
 }
 `;

spec/unit/http-api/fetch.spec.ts

@@ -14,11 +14,22 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { Mocked } from "jest-mock";
+
 import { FetchHttpApi } from "../../../src/http-api/fetch";
 import { TypedEventEmitter } from "../../../src/models/typed-event-emitter";
-import { ClientPrefix, HttpApiEvent, HttpApiEventHandlerMap, IdentityPrefix, IHttpOpts, Method } from "../../../src";
+import {
+    ClientPrefix,
+    HttpApiEvent,
+    HttpApiEventHandlerMap,
+    IdentityPrefix,
+    IHttpOpts,
+    MatrixError,
+    Method,
+} from "../../../src";
 import { emitPromise } from "../../test-utils/test-utils";
-import { QueryDict } from "../../../src/utils";
+import { defer, QueryDict } from "../../../src/utils";
+import { Logger } from "../../../src/logger";
 
 describe("FetchHttpApi", () => {
     const baseUrl = "http://baseUrl";
@@ -228,13 +239,145 @@ describe("FetchHttpApi", () => {
     });
 
     describe("authedRequest", () => {
-        it("should not include token if unset", () => {
-            const fetchFn = jest.fn();
+        it("should not include token if unset", async () => {
+            const fetchFn = jest.fn().mockResolvedValue({ ok: true });
             const emitter = new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>();
             const api = new FetchHttpApi(emitter, { baseUrl, prefix, fetchFn });
-            api.authedRequest(Method.Post, "/account/password");
+            await api.authedRequest(Method.Post, "/account/password");
             expect(fetchFn.mock.calls[0][1].headers.Authorization).toBeUndefined();
         });
+
+        describe("with refresh token", () => {
+            const accessToken = "test-access-token";
+            const refreshToken = "test-refresh-token";
+
+            describe("when an unknown token error is encountered", () => {
+                const unknownTokenErrBody = {
+                    errcode: "M_UNKNOWN_TOKEN",
+                    error: "Token is not active",
+                    soft_logout: false,
+                };
+                const unknownTokenErr = new MatrixError(unknownTokenErrBody, 401);
+                const unknownTokenResponse = {
+                    ok: false,
+                    status: 401,
+                    headers: {
+                        get(name: string): string | null {
+                            return name === "Content-Type" ? "application/json" : null;
+                        },
+                    },
+                    text: jest.fn().mockResolvedValue(JSON.stringify(unknownTokenErrBody)),
+                };
+                const okayResponse = {
+                    ok: true,
+                    status: 200,
+                };
+
+                describe("without a tokenRefreshFunction", () => {
+                    it("should emit logout and throw", async () => {
+                        const fetchFn = jest.fn().mockResolvedValue(unknownTokenResponse);
+                        const emitter = new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>();
+                        jest.spyOn(emitter, "emit");
+                        const api = new FetchHttpApi(emitter, { baseUrl, prefix, fetchFn, accessToken, refreshToken });
+                        await expect(api.authedRequest(Method.Post, "/account/password")).rejects.toThrow(
+                            unknownTokenErr,
+                        );
+                        expect(emitter.emit).toHaveBeenCalledWith(HttpApiEvent.SessionLoggedOut, unknownTokenErr);
+                    });
+                });
+
+                describe("with a tokenRefreshFunction", () => {
+                    it("should emit logout and throw when token refresh fails", async () => {
+                        const error = new Error("uh oh");
+                        const tokenRefreshFunction = jest.fn().mockRejectedValue(error);
+                        const fetchFn = jest.fn().mockResolvedValue(unknownTokenResponse);
+                        const emitter = new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>();
+                        jest.spyOn(emitter, "emit");
+                        const api = new FetchHttpApi(emitter, {
+                            baseUrl,
+                            prefix,
+                            fetchFn,
+                            tokenRefreshFunction,
+                            accessToken,
+                            refreshToken,
+                        });
+                        await expect(api.authedRequest(Method.Post, "/account/password")).rejects.toThrow(
+                            unknownTokenErr,
+                        );
+                        expect(tokenRefreshFunction).toHaveBeenCalledWith(refreshToken);
+                        expect(emitter.emit).toHaveBeenCalledWith(HttpApiEvent.SessionLoggedOut, unknownTokenErr);
+                    });
+
+                    it("should refresh token and retry request", async () => {
+                        const newAccessToken = "new-access-token";
+                        const newRefreshToken = "new-refresh-token";
+                        const tokenRefreshFunction = jest.fn().mockResolvedValue({
+                            accessToken: newAccessToken,
+                            refreshToken: newRefreshToken,
+                        });
+                        const fetchFn = jest
+                            .fn()
+                            .mockResolvedValueOnce(unknownTokenResponse)
+                            .mockResolvedValueOnce(okayResponse);
+                        const emitter = new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>();
+                        jest.spyOn(emitter, "emit");
+                        const api = new FetchHttpApi(emitter, {
+                            baseUrl,
+                            prefix,
+                            fetchFn,
+                            tokenRefreshFunction,
+                            accessToken,
+                            refreshToken,
+                        });
+                        const result = await api.authedRequest(Method.Post, "/account/password");
+                        expect(result).toEqual(okayResponse);
+                        expect(tokenRefreshFunction).toHaveBeenCalledWith(refreshToken);
+
+                        expect(fetchFn).toHaveBeenCalledTimes(2);
+                        // uses new access token
+                        expect(fetchFn.mock.calls[1][1].headers.Authorization).toEqual("Bearer new-access-token");
+                        expect(emitter.emit).not.toHaveBeenCalledWith(HttpApiEvent.SessionLoggedOut, unknownTokenErr);
+                    });
+
+                    it("should only try to refresh the token once", async () => {
+                        const newAccessToken = "new-access-token";
+                        const newRefreshToken = "new-refresh-token";
+                        const tokenRefreshFunction = jest.fn().mockResolvedValue({
+                            accessToken: newAccessToken,
+                            refreshToken: newRefreshToken,
+                        });
+
+                        // fetch doesn't like our new or old tokens
+                        const fetchFn = jest.fn().mockResolvedValue(unknownTokenResponse);
+
+                        const emitter = new TypedEventEmitter<HttpApiEvent, HttpApiEventHandlerMap>();
+                        jest.spyOn(emitter, "emit");
+                        const api = new FetchHttpApi(emitter, {
+                            baseUrl,
+                            prefix,
+                            fetchFn,
+                            tokenRefreshFunction,
+                            accessToken,
+                            refreshToken,
+                        });
+                        await expect(api.authedRequest(Method.Post, "/account/password")).rejects.toThrow(
+                            unknownTokenErr,
+                        );
+
+                        // tried to refresh the token once
+                        expect(tokenRefreshFunction).toHaveBeenCalledWith(refreshToken);
+                        expect(tokenRefreshFunction).toHaveBeenCalledTimes(1);
+
+                        expect(fetchFn).toHaveBeenCalledTimes(2);
+                        // uses new access token on retry
+                        expect(fetchFn.mock.calls[1][1].headers.Authorization).toEqual("Bearer new-access-token");
+
+                        // logged out after refreshed access token is rejected
+                        expect(emitter.emit).toHaveBeenCalledWith(HttpApiEvent.SessionLoggedOut, unknownTokenErr);
+                    });
+                });
+            });
+        });
     });
 
     describe("getUrl()", () => {
@@ -274,11 +417,13 @@ describe("FetchHttpApi", () => {
         ];
         const runTests = (fetchBaseUrl: string) => {
             it.each<TestCase>(testCases)(
-                "creates url with params %s",
-                ({ path, queryParams, prefix, baseUrl }, result) => {
+                "creates url with params %s => %s",
+                ({ path, queryParams, prefix, baseUrl }, expected) => {
                     const api = makeApi(fetchBaseUrl);
 
-                    expect(api.getUrl(path, queryParams, prefix, baseUrl)).toEqual(new URL(result));
+                    const result = api.getUrl(path, queryParams, prefix, baseUrl);
+                    // we only check the stringified URL, to avoid having the test depend on the internals of URL.
+                    expect(result.toString()).toEqual(expected);
                 },
             );
         };
@@ -290,4 +435,37 @@ describe("FetchHttpApi", () => {
             runTests(baseUrlWithTrailingSlash);
         });
     });
+
+    it("should not log query parameters", async () => {
+        jest.useFakeTimers();
+        const deferred = defer<Response>();
+        const fetchFn = jest.fn().mockReturnValue(deferred.promise);
+        const mockLogger = {
+            debug: jest.fn(),
+        } as unknown as Mocked<Logger>;
+        const api = new FetchHttpApi(new TypedEventEmitter<any, any>(), {
+            baseUrl,
+            prefix,
+            fetchFn,
+            logger: mockLogger,
+        });
+        const prom = api.requestOtherUrl(Method.Get, "https://server:8448/some/path?query=param#fragment");
+        jest.advanceTimersByTime(1234);
+        deferred.resolve({ ok: true, status: 200, text: () => Promise.resolve("RESPONSE") } as Response);
+        await prom;
+        expect(mockLogger.debug).not.toHaveBeenCalledWith("fragment");
+        expect(mockLogger.debug).not.toHaveBeenCalledWith("query");
+        expect(mockLogger.debug).not.toHaveBeenCalledWith("param");
+        expect(mockLogger.debug).toHaveBeenCalledTimes(2);
+        expect(mockLogger.debug.mock.calls[0]).toMatchInlineSnapshot(`
+            [
+              "FetchHttpApi: --> GET https://server:8448/some/path?query=xxx",
+            ]
+        `);
+        expect(mockLogger.debug.mock.calls[1]).toMatchInlineSnapshot(`
+            [
+              "FetchHttpApi: <-- GET https://server:8448/some/path?query=xxx [1234ms 200]",
+            ]
+        `);
+    });
 });

spec/unit/http-api/index.spec.ts

@@ -84,7 +84,7 @@ describe("MatrixHttpApi", () => {
         upload = api.uploadContent({} as File);
         expect(xhr.open).toHaveBeenCalledWith(
             Method.Post,
-            baseUrl.toLowerCase() + "/_matrix/media/r0/upload?access_token=token",
+            baseUrl.toLowerCase() + "/_matrix/media/v3/upload?access_token=token",
         );
         expect(xhr.setRequestHeader).not.toHaveBeenCalledWith("Authorization");
     });
@@ -96,7 +96,7 @@ describe("MatrixHttpApi", () => {
             accessToken: "token",
         });
         upload = api.uploadContent({} as File);
-        expect(xhr.open).toHaveBeenCalledWith(Method.Post, baseUrl.toLowerCase() + "/_matrix/media/r0/upload");
+        expect(xhr.open).toHaveBeenCalledWith(Method.Post, baseUrl.toLowerCase() + "/_matrix/media/v3/upload");
         expect(xhr.setRequestHeader).toHaveBeenCalledWith("Authorization", "Bearer token");
     });
 
@@ -105,14 +105,14 @@ describe("MatrixHttpApi", () => {
         upload = api.uploadContent({} as File, { name: "name" });
         expect(xhr.open).toHaveBeenCalledWith(
             Method.Post,
-            baseUrl.toLowerCase() + "/_matrix/media/r0/upload?filename=name",
+            baseUrl.toLowerCase() + "/_matrix/media/v3/upload?filename=name",
         );
     });
 
     it("should allow not sending the filename", () => {
         const api = new MatrixHttpApi(new TypedEventEmitter<any, any>(), { baseUrl, prefix });
         upload = api.uploadContent({} as File, { name: "name", includeFilename: false });
-        expect(xhr.open).toHaveBeenCalledWith(Method.Post, baseUrl.toLowerCase() + "/_matrix/media/r0/upload");
+        expect(xhr.open).toHaveBeenCalledWith(Method.Post, baseUrl.toLowerCase() + "/_matrix/media/v3/upload");
     });
 
     it("should abort xhr when the upload is aborted", () => {

spec/unit/interactive-auth.spec.ts

@@ -94,7 +94,6 @@ describe("InteractiveAuth", () => {
             authData: {
                 session: "sessionId",
                 flows: [{ stages: [AuthType.Password] }],
-                errcode: "MockError0",
                 params: {
                     [AuthType.Password]: { param: "aa" },
                 },
@@ -376,7 +375,7 @@ describe("InteractiveAuth", () => {
         await expect(ia.attemptAuth.bind(ia)).rejects.toThrow(new Error("No appropriate authentication flow found"));
     });
 
-    it("should handle unexpected error types without data propery set", async () => {
+    it("should handle unexpected error types without data property set", async () => {
         const doRequest = jest.fn();
         const stateUpdated = jest.fn();
         const requestEmailToken = jest.fn();
@@ -560,4 +559,45 @@ describe("InteractiveAuth", () => {
         ia.chooseStage();
         expect(ia.getChosenFlow()?.stages).toEqual([AuthType.Password]);
     });
+
+    it("should fire stateUpdated callback with error when a request fails", async () => {
+        const doRequest = jest.fn();
+        const stateUpdated = jest.fn();
+
+        const ia = new InteractiveAuth({
+            matrixClient: getFakeClient(),
+            doRequest: doRequest,
+            stateUpdated: stateUpdated,
+            requestEmailToken: jest.fn(),
+            authData: {
+                session: "sessionId",
+                flows: [{ stages: [AuthType.Password] }],
+                params: {
+                    [AuthType.Password]: { param: "aa" },
+                },
+            },
+        });
+
+        // StateUpdated should be called. We call submitAuthDict() to trigger a request ...
+        let firstTime = true;
+        stateUpdated.mockImplementation((stage) => {
+            expect(stage).toEqual(AuthType.Password);
+            // Only trigger the request the first time, to avoid an infinite loop
+            if (firstTime) {
+                firstTime = false;
+                ia.submitAuthDict({
+                    type: AuthType.Password,
+                });
+            }
+        });
+
+        // .. which which we then reject, so we can test the behaviour in that case.
+        doRequest.mockRejectedValue(new MatrixError({ errcode: "M_UNKNOWN", error: "This is an error" }));
+
+        await Promise.allSettled([ia.attemptAuth()]);
+        expect(stateUpdated).toHaveBeenCalledWith("m.login.password", {
+            errcode: "M_UNKNOWN",
+            error: "This is an error",
+        });
+    });
 });

spec/unit/login.spec.ts

@@ -121,7 +121,7 @@ describe("refreshToken", () => {
             body: { errcode: "M_UNRECOGNIZED" },
         });
 
-        expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ errcode: "M_UNRECOGNIZED" });
+        await expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ errcode: "M_UNRECOGNIZED" });
     });
 
     it("re-raises non-M_UNRECOGNIZED exceptions from /v3", async () => {
@@ -132,7 +132,7 @@ describe("refreshToken", () => {
             throw new Error("/v1/refresh unexpectedly called");
         });
 
-        expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ httpStatus: 429 });
+        await expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ httpStatus: 429 });
     });
 
     it("re-raises non-M_UNRECOGNIZED exceptions from /v1", async () => {
@@ -144,6 +144,6 @@ describe("refreshToken", () => {
         });
         fetchMock.postOnce(client.http.getUrl("/refresh", undefined, ClientPrefix.V1).toString(), 429);
 
-        expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ httpStatus: 429 });
+        await expect(client.refreshToken("initial_refresh_token")).rejects.toMatchObject({ httpStatus: 429 });
     });
 });

spec/unit/matrix-client.spec.ts

@@ -201,7 +201,7 @@ describe("MatrixClient", function () {
         if (path === KEEP_ALIVE_PATH && acceptKeepalives) {
             return Promise.resolve({
                 unstable_features: unstableFeatures,
-                versions: ["r0.6.0", "r0.6.1"],
+                versions: ["v1.1"],
             });
         }
         const next = httpLookups.shift();
@@ -328,6 +328,7 @@ describe("MatrixClient", function () {
                 "storeFilter",
                 "startup",
                 "deleteAllData",
+                "setUserCreator",
             ] as const
         ).reduce((r, k) => {
             r[k] = jest.fn();
@@ -2266,7 +2267,6 @@ describe("MatrixClient", function () {
         function roomCreateEvent(newRoomId: string, predecessorRoomId: string): MatrixEvent {
             return new MatrixEvent({
                 content: {
-                    "creator": "@daryl:alexandria.example.com",
                     "m.federate": true,
                     "predecessor": {
                         event_id: "id_of_last_event",

spec/unit/matrixrtc/CallMembership.spec.ts

@@ -0,0 +1,138 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MatrixEvent } from "../../../src";
+import { CallMembership, CallMembershipData } from "../../../src/matrixrtc/CallMembership";
+
+const membershipTemplate: CallMembershipData = {
+    call_id: "",
+    scope: "m.room",
+    application: "m.call",
+    device_id: "AAAAAAA",
+    expires: 5000,
+    membershipID: "bloop",
+};
+
+function makeMockEvent(originTs = 0): MatrixEvent {
+    return {
+        getTs: jest.fn().mockReturnValue(originTs),
+        getSender: jest.fn().mockReturnValue("@alice:example.org"),
+    } as unknown as MatrixEvent;
+}
+
+describe("CallMembership", () => {
+    it("rejects membership with no expiry", () => {
+        expect(() => {
+            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { expires: undefined }));
+        }).toThrow();
+    });
+
+    it("rejects membership with no device_id", () => {
+        expect(() => {
+            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { device_id: undefined }));
+        }).toThrow();
+    });
+
+    it("rejects membership with no call_id", () => {
+        expect(() => {
+            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { call_id: undefined }));
+        }).toThrow();
+    });
+
+    it("rejects membership with no scope", () => {
+        expect(() => {
+            new CallMembership(makeMockEvent(), Object.assign({}, membershipTemplate, { scope: undefined }));
+        }).toThrow();
+    });
+
+    it("uses event timestamp if no created_ts", () => {
+        const membership = new CallMembership(makeMockEvent(12345), membershipTemplate);
+        expect(membership.createdTs()).toEqual(12345);
+    });
+
+    it("uses created_ts if present", () => {
+        const membership = new CallMembership(
+            makeMockEvent(12345),
+            Object.assign({}, membershipTemplate, { created_ts: 67890 }),
+        );
+        expect(membership.createdTs()).toEqual(67890);
+    });
+
+    it("computes absolute expiry time", () => {
+        const membership = new CallMembership(makeMockEvent(1000), membershipTemplate);
+        expect(membership.getAbsoluteExpiry()).toEqual(5000 + 1000);
+    });
+
+    it("considers memberships unexpired if local age low enough", () => {
+        const fakeEvent = makeMockEvent(1000);
+        fakeEvent.getLocalAge = jest.fn().mockReturnValue(3000);
+        const membership = new CallMembership(fakeEvent, membershipTemplate);
+        expect(membership.isExpired()).toEqual(false);
+    });
+
+    it("considers memberships expired when local age large", () => {
+        const fakeEvent = makeMockEvent(1000);
+        fakeEvent.localTimestamp = Date.now() - 6000;
+        const membership = new CallMembership(fakeEvent, membershipTemplate);
+        expect(membership.isExpired()).toEqual(true);
+    });
+
+    it("returns active foci", () => {
+        const fakeEvent = makeMockEvent();
+        const mockFocus = { type: "this_is_a_mock_focus" };
+        const membership = new CallMembership(
+            fakeEvent,
+            Object.assign({}, membershipTemplate, { foci_active: [mockFocus] }),
+        );
+        expect(membership.getActiveFoci()).toEqual([mockFocus]);
+    });
+
+    describe("expiry calculation", () => {
+        let fakeEvent: MatrixEvent;
+        let membership: CallMembership;
+
+        beforeEach(() => {
+            // server origin timestamp for this event is 1000
+            fakeEvent = makeMockEvent(1000);
+            // our clock would have been at 2000 at the creation time (our clock at event receive time - age)
+            // (ie. the local clock is 1 second ahead of the servers' clocks)
+            fakeEvent.localTimestamp = 2000;
+
+            // for simplicity's sake, we say that the event's age is zero
+            fakeEvent.getLocalAge = jest.fn().mockReturnValue(0);
+
+            membership = new CallMembership(fakeEvent!, membershipTemplate);
+
+            jest.useFakeTimers();
+        });
+
+        afterEach(() => {
+            jest.useRealTimers();
+        });
+
+        it("converts expiry time into local clock", () => {
+            // for sanity's sake, make sure the server-relative expiry time is what we expect
+            expect(membership.getAbsoluteExpiry()).toEqual(6000);
+            // therefore the expiry time converted to our clock should be 1 second later
+            expect(membership.getLocalExpiry()).toEqual(7000);
+        });
+
+        it("calculates time until expiry", () => {
+            jest.setSystemTime(2000);
+            expect(membership.getMsUntilExpiry()).toEqual(5000);
+        });
+    });
+});

spec/unit/matrixrtc/MatrixRTCSession.spec.ts

@@ -0,0 +1,718 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventTimeline, EventType, MatrixClient, MatrixError, MatrixEvent, Room } from "../../../src";
+import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
+import { MatrixRTCSession, MatrixRTCSessionEvent } from "../../../src/matrixrtc/MatrixRTCSession";
+import { EncryptionKeysEventContent } from "../../../src/matrixrtc/types";
+import { randomString } from "../../../src/randomstring";
+import { makeMockRoom, makeMockRoomState, mockRTCEvent } from "./mocks";
+
+const membershipTemplate: CallMembershipData = {
+    call_id: "",
+    scope: "m.room",
+    application: "m.call",
+    device_id: "AAAAAAA",
+    expires: 60 * 60 * 1000,
+    membershipID: "bloop",
+};
+
+const mockFocus = { type: "mock" };
+
+describe("MatrixRTCSession", () => {
+    let client: MatrixClient;
+    let sess: MatrixRTCSession | undefined;
+
+    beforeEach(() => {
+        client = new MatrixClient({ baseUrl: "base_url" });
+        client.getUserId = jest.fn().mockReturnValue("@alice:example.org");
+        client.getDeviceId = jest.fn().mockReturnValue("AAAAAAA");
+    });
+
+    afterEach(() => {
+        client.stopClient();
+        client.matrixRTC.stop();
+        if (sess) sess.stop();
+        sess = undefined;
+    });
+
+    it("Creates a room-scoped session from room state", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess?.memberships.length).toEqual(1);
+        expect(sess?.memberships[0].callId).toEqual("");
+        expect(sess?.memberships[0].scope).toEqual("m.room");
+        expect(sess?.memberships[0].application).toEqual("m.call");
+        expect(sess?.memberships[0].deviceId).toEqual("AAAAAAA");
+        expect(sess?.memberships[0].membershipID).toEqual("bloop");
+        expect(sess?.memberships[0].isExpired()).toEqual(false);
+        expect(sess?.callId).toEqual("");
+    });
+
+    it("ignores expired memberships events", () => {
+        const expiredMembership = Object.assign({}, membershipTemplate);
+        expiredMembership.expires = 1000;
+        expiredMembership.device_id = "EXPIRED";
+        const mockRoom = makeMockRoom([membershipTemplate, expiredMembership], 10000);
+
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess?.memberships.length).toEqual(1);
+        expect(sess?.memberships[0].deviceId).toEqual("AAAAAAA");
+    });
+
+    it("honours created_ts", () => {
+        const expiredMembership = Object.assign({}, membershipTemplate);
+        expiredMembership.created_ts = 500;
+        expiredMembership.expires = 1000;
+        const mockRoom = makeMockRoom([expiredMembership]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess?.memberships[0].getAbsoluteExpiry()).toEqual(1500);
+    });
+
+    it("returns empty session if no membership events are present", () => {
+        const mockRoom = makeMockRoom([]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess?.memberships).toHaveLength(0);
+    });
+
+    it("safely ignores events with no memberships section", () => {
+        const mockRoom = {
+            roomId: randomString(8),
+            getLiveTimeline: jest.fn().mockReturnValue({
+                getState: jest.fn().mockReturnValue({
+                    getStateEvents: (_type: string, _stateKey: string) => [
+                        {
+                            getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix),
+                            getContent: jest.fn().mockReturnValue({}),
+                            getSender: jest.fn().mockReturnValue("@mock:user.example"),
+                            getTs: jest.fn().mockReturnValue(1000),
+                            getLocalAge: jest.fn().mockReturnValue(0),
+                        },
+                    ],
+                }),
+            }),
+        };
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom as unknown as Room);
+        expect(sess.memberships).toHaveLength(0);
+    });
+
+    it("safely ignores events with junk memberships section", () => {
+        const mockRoom = {
+            roomId: randomString(8),
+            getLiveTimeline: jest.fn().mockReturnValue({
+                getState: jest.fn().mockReturnValue({
+                    getStateEvents: (_type: string, _stateKey: string) => [
+                        {
+                            getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix),
+                            getContent: jest.fn().mockReturnValue({ memberships: "i am a fish" }),
+                            getSender: jest.fn().mockReturnValue("@mock:user.example"),
+                            getTs: jest.fn().mockReturnValue(1000),
+                            getLocalAge: jest.fn().mockReturnValue(0),
+                        },
+                    ],
+                }),
+            }),
+        };
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom as unknown as Room);
+        expect(sess.memberships).toHaveLength(0);
+    });
+
+    it("ignores memberships with no expires_ts", () => {
+        const expiredMembership = Object.assign({}, membershipTemplate);
+        (expiredMembership.expires as number | undefined) = undefined;
+        const mockRoom = makeMockRoom([expiredMembership]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess.memberships).toHaveLength(0);
+    });
+
+    it("ignores memberships with no device_id", () => {
+        const testMembership = Object.assign({}, membershipTemplate);
+        (testMembership.device_id as string | undefined) = undefined;
+        const mockRoom = makeMockRoom([testMembership]);
+        const sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess.memberships).toHaveLength(0);
+    });
+
+    it("ignores memberships with no call_id", () => {
+        const testMembership = Object.assign({}, membershipTemplate);
+        (testMembership.call_id as string | undefined) = undefined;
+        const mockRoom = makeMockRoom([testMembership]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess.memberships).toHaveLength(0);
+    });
+
+    it("ignores memberships with no scope", () => {
+        const testMembership = Object.assign({}, membershipTemplate);
+        (testMembership.scope as string | undefined) = undefined;
+        const mockRoom = makeMockRoom([testMembership]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess.memberships).toHaveLength(0);
+    });
+
+    it("ignores anything that's not a room-scoped call (for now)", () => {
+        const testMembership = Object.assign({}, membershipTemplate);
+        testMembership.scope = "m.user";
+        const mockRoom = makeMockRoom([testMembership]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        expect(sess.memberships).toHaveLength(0);
+    });
+
+    describe("getOldestMembership", () => {
+        it("returns the oldest membership event", () => {
+            const mockRoom = makeMockRoom([
+                Object.assign({}, membershipTemplate, { device_id: "foo", created_ts: 3000 }),
+                Object.assign({}, membershipTemplate, { device_id: "old", created_ts: 1000 }),
+                Object.assign({}, membershipTemplate, { device_id: "bar", created_ts: 2000 }),
+            ]);
+
+            sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+            expect(sess.getOldestMembership()!.deviceId).toEqual("old");
+        });
+    });
+
+    describe("joining", () => {
+        let mockRoom: Room;
+        let sendStateEventMock: jest.Mock;
+        let sendEventMock: jest.Mock;
+
+        beforeEach(() => {
+            sendStateEventMock = jest.fn();
+            sendEventMock = jest.fn();
+            client.sendStateEvent = sendStateEventMock;
+            client.sendEvent = sendEventMock;
+
+            mockRoom = makeMockRoom([]);
+            sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        });
+
+        afterEach(() => {
+            // stop the timers
+            sess!.leaveRoomSession();
+        });
+
+        it("starts un-joined", () => {
+            expect(sess!.isJoined()).toEqual(false);
+        });
+
+        it("shows joined once join is called", () => {
+            sess!.joinRoomSession([mockFocus]);
+            expect(sess!.isJoined()).toEqual(true);
+        });
+
+        it("sends a membership event when joining a call", () => {
+            sess!.joinRoomSession([mockFocus]);
+
+            expect(client.sendStateEvent).toHaveBeenCalledWith(
+                mockRoom!.roomId,
+                EventType.GroupCallMemberPrefix,
+                {
+                    memberships: [
+                        {
+                            application: "m.call",
+                            scope: "m.room",
+                            call_id: "",
+                            device_id: "AAAAAAA",
+                            expires: 3600000,
+                            foci_active: [{ type: "mock" }],
+                            membershipID: expect.stringMatching(".*"),
+                        },
+                    ],
+                },
+                "@alice:example.org",
+            );
+        });
+
+        it("does nothing if join called when already joined", () => {
+            sess!.joinRoomSession([mockFocus]);
+
+            expect(client.sendStateEvent).toHaveBeenCalledTimes(1);
+
+            sess!.joinRoomSession([mockFocus]);
+            expect(client.sendStateEvent).toHaveBeenCalledTimes(1);
+        });
+
+        it("renews membership event before expiry time", async () => {
+            jest.useFakeTimers();
+            let resolveFn: ((_roomId: string, _type: string, val: Record<string, any>) => void) | undefined;
+
+            const eventSentPromise = new Promise<Record<string, any>>((r) => {
+                resolveFn = (_roomId: string, _type: string, val: Record<string, any>) => {
+                    r(val);
+                };
+            });
+            try {
+                const sendStateEventMock = jest.fn().mockImplementation(resolveFn);
+                client.sendStateEvent = sendStateEventMock;
+
+                sess!.joinRoomSession([mockFocus]);
+
+                const eventContent = await eventSentPromise;
+
+                // definitely should have renewed by 1 second before the expiry!
+                const timeElapsed = 60 * 60 * 1000 - 1000;
+                mockRoom.getLiveTimeline().getState(EventTimeline.FORWARDS)!.getStateEvents = jest
+                    .fn()
+                    .mockReturnValue(mockRTCEvent(eventContent.memberships, mockRoom.roomId, timeElapsed));
+
+                const eventReSentPromise = new Promise<Record<string, any>>((r) => {
+                    resolveFn = (_roomId: string, _type: string, val: Record<string, any>) => {
+                        r(val);
+                    };
+                });
+
+                sendStateEventMock.mockReset().mockImplementation(resolveFn);
+
+                jest.setSystemTime(Date.now() + timeElapsed);
+                jest.advanceTimersByTime(timeElapsed);
+                await eventReSentPromise;
+
+                expect(sendStateEventMock).toHaveBeenCalledWith(
+                    mockRoom.roomId,
+                    EventType.GroupCallMemberPrefix,
+                    {
+                        memberships: [
+                            {
+                                application: "m.call",
+                                scope: "m.room",
+                                call_id: "",
+                                device_id: "AAAAAAA",
+                                expires: 3600000 * 2,
+                                foci_active: [{ type: "mock" }],
+                                created_ts: 1000,
+                                membershipID: expect.stringMatching(".*"),
+                            },
+                        ],
+                    },
+                    "@alice:example.org",
+                );
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("creates a key when joining", () => {
+            sess!.joinRoomSession([mockFocus], true);
+            const keys = sess?.getKeysForParticipant("@alice:example.org", "AAAAAAA");
+            expect(keys).toHaveLength(1);
+
+            const allKeys = sess!.getEncryptionKeys();
+            expect(allKeys).toBeTruthy();
+            expect(Array.from(allKeys)).toHaveLength(1);
+        });
+
+        it("sends keys when joining", async () => {
+            const eventSentPromise = new Promise((resolve) => {
+                sendEventMock.mockImplementation(resolve);
+            });
+
+            sess!.joinRoomSession([mockFocus], true);
+
+            await eventSentPromise;
+
+            expect(sendEventMock).toHaveBeenCalledWith(expect.stringMatching(".*"), "io.element.call.encryption_keys", {
+                call_id: "",
+                device_id: "AAAAAAA",
+                keys: [
+                    {
+                        index: 0,
+                        key: expect.stringMatching(".*"),
+                    },
+                ],
+            });
+        });
+
+        it("retries key sends", async () => {
+            jest.useFakeTimers();
+            let firstEventSent = false;
+
+            try {
+                const eventSentPromise = new Promise<void>((resolve) => {
+                    sendEventMock.mockImplementation(() => {
+                        if (!firstEventSent) {
+                            jest.advanceTimersByTime(10000);
+
+                            firstEventSent = true;
+                            const e = new Error() as MatrixError;
+                            e.data = {};
+                            throw e;
+                        } else {
+                            resolve();
+                        }
+                    });
+                });
+
+                sess!.joinRoomSession([mockFocus], true);
+                jest.advanceTimersByTime(10000);
+
+                await eventSentPromise;
+
+                expect(sendEventMock).toHaveBeenCalledTimes(2);
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("cancels key send event that fail", async () => {
+            const eventSentinel = {} as unknown as MatrixEvent;
+
+            client.cancelPendingEvent = jest.fn();
+            sendEventMock.mockImplementation(() => {
+                const e = new Error() as MatrixError;
+                e.data = {};
+                e.event = eventSentinel;
+                throw e;
+            });
+
+            sess!.joinRoomSession([mockFocus], true);
+
+            expect(client.cancelPendingEvent).toHaveBeenCalledWith(eventSentinel);
+        });
+
+        it("Re-sends key if a new member joins", async () => {
+            jest.useFakeTimers();
+            try {
+                const mockRoom = makeMockRoom([membershipTemplate]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const keysSentPromise1 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                await keysSentPromise1;
+
+                sendEventMock.mockClear();
+                jest.advanceTimersByTime(10000);
+
+                const keysSentPromise2 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                const onMembershipsChanged = jest.fn();
+                sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate, member2], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                await keysSentPromise2;
+
+                expect(sendEventMock).toHaveBeenCalled();
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("Rotates key if a member leaves", async () => {
+            jest.useFakeTimers();
+            try {
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+                const mockRoom = makeMockRoom([membershipTemplate, member2]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const onMyEncryptionKeyChanged = jest.fn();
+                sess.on(
+                    MatrixRTCSessionEvent.EncryptionKeyChanged,
+                    (_key: Uint8Array, _idx: number, participantId: string) => {
+                        if (participantId === `${client.getUserId()}:${client.getDeviceId()}`) {
+                            onMyEncryptionKeyChanged();
+                        }
+                    },
+                );
+
+                const keysSentPromise1 = new Promise<EncryptionKeysEventContent>((resolve) => {
+                    sendEventMock.mockImplementation((_roomId, _evType, payload) => resolve(payload));
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                const firstKeysPayload = await keysSentPromise1;
+                expect(firstKeysPayload.keys).toHaveLength(1);
+
+                sendEventMock.mockClear();
+
+                const keysSentPromise2 = new Promise<EncryptionKeysEventContent>((resolve) => {
+                    sendEventMock.mockImplementation((_roomId, _evType, payload) => resolve(payload));
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                jest.advanceTimersByTime(10000);
+
+                const secondKeysPayload = await keysSentPromise2;
+
+                expect(secondKeysPayload.keys).toHaveLength(2);
+                expect(onMyEncryptionKeyChanged).toHaveBeenCalledTimes(2);
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+
+        it("Doesn't re-send key immediately", async () => {
+            const realSetImmediate = setImmediate;
+            jest.useFakeTimers();
+            try {
+                const mockRoom = makeMockRoom([membershipTemplate]);
+                sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+                const keysSentPromise1 = new Promise((resolve) => {
+                    sendEventMock.mockImplementation(resolve);
+                });
+
+                sess.joinRoomSession([mockFocus], true);
+                await keysSentPromise1;
+
+                sendEventMock.mockClear();
+
+                const onMembershipsChanged = jest.fn();
+                sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+                const member2 = Object.assign({}, membershipTemplate, {
+                    device_id: "BBBBBBB",
+                });
+
+                mockRoom.getLiveTimeline().getState = jest
+                    .fn()
+                    .mockReturnValue(makeMockRoomState([membershipTemplate, member2], mockRoom.roomId, undefined));
+                sess.onMembershipUpdate();
+
+                await new Promise((resolve) => {
+                    realSetImmediate(resolve);
+                });
+
+                expect(sendEventMock).not.toHaveBeenCalled();
+            } finally {
+                jest.useRealTimers();
+            }
+        });
+    });
+
+    it("Does not emits if no membership changes", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+        const onMembershipsChanged = jest.fn();
+        sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+        sess.onMembershipUpdate();
+
+        expect(onMembershipsChanged).not.toHaveBeenCalled();
+    });
+
+    it("Emits on membership changes", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+        const onMembershipsChanged = jest.fn();
+        sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+        mockRoom.getLiveTimeline().getState = jest
+            .fn()
+            .mockReturnValue(makeMockRoomState([], mockRoom.roomId, undefined));
+        sess.onMembershipUpdate();
+
+        expect(onMembershipsChanged).toHaveBeenCalled();
+    });
+
+    it("emits an event at the time a membership event expires", () => {
+        jest.useFakeTimers();
+        try {
+            const membership = Object.assign({}, membershipTemplate);
+            const mockRoom = makeMockRoom([membership], 0);
+
+            sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+            const membershipObject = sess.memberships[0];
+
+            const onMembershipsChanged = jest.fn();
+            sess.on(MatrixRTCSessionEvent.MembershipsChanged, onMembershipsChanged);
+
+            jest.advanceTimersByTime(61 * 1000 * 1000);
+
+            expect(onMembershipsChanged).toHaveBeenCalledWith([membershipObject], []);
+            expect(sess?.memberships.length).toEqual(0);
+        } finally {
+            jest.useRealTimers();
+        }
+    });
+
+    it("prunes expired memberships on update", () => {
+        jest.useFakeTimers();
+        try {
+            client.sendStateEvent = jest.fn();
+
+            const mockMemberships = [
+                Object.assign({}, membershipTemplate, {
+                    device_id: "OTHERDEVICE",
+                    expires: 1000,
+                }),
+            ];
+            const mockRoomNoExpired = makeMockRoom(mockMemberships, 0);
+
+            sess = MatrixRTCSession.roomSessionForRoom(client, mockRoomNoExpired);
+
+            // sanity check
+            expect(sess.memberships).toHaveLength(1);
+            expect(sess.memberships[0].deviceId).toEqual("OTHERDEVICE");
+
+            jest.advanceTimersByTime(10000);
+
+            sess.joinRoomSession([mockFocus]);
+
+            expect(client.sendStateEvent).toHaveBeenCalledWith(
+                mockRoomNoExpired!.roomId,
+                EventType.GroupCallMemberPrefix,
+                {
+                    memberships: [
+                        {
+                            application: "m.call",
+                            scope: "m.room",
+                            call_id: "",
+                            device_id: "AAAAAAA",
+                            expires: 3600000,
+                            foci_active: [mockFocus],
+                            membershipID: expect.stringMatching(".*"),
+                        },
+                    ],
+                },
+                "@alice:example.org",
+            );
+        } finally {
+            jest.useRealTimers();
+        }
+    });
+
+    it("fills in created_ts for other memberships on update", () => {
+        client.sendStateEvent = jest.fn();
+
+        const mockRoom = makeMockRoom([
+            Object.assign({}, membershipTemplate, {
+                device_id: "OTHERDEVICE",
+            }),
+        ]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+
+        sess.joinRoomSession([mockFocus]);
+
+        expect(client.sendStateEvent).toHaveBeenCalledWith(
+            mockRoom!.roomId,
+            EventType.GroupCallMemberPrefix,
+            {
+                memberships: [
+                    {
+                        application: "m.call",
+                        scope: "m.room",
+                        call_id: "",
+                        device_id: "OTHERDEVICE",
+                        expires: 3600000,
+                        created_ts: 1000,
+                        membershipID: expect.stringMatching(".*"),
+                    },
+                    {
+                        application: "m.call",
+                        scope: "m.room",
+                        call_id: "",
+                        device_id: "AAAAAAA",
+                        expires: 3600000,
+                        foci_active: [mockFocus],
+                        membershipID: expect.stringMatching(".*"),
+                    },
+                ],
+            },
+            "@alice:example.org",
+        );
+    });
+
+    it("collects keys from encryption events", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: "bobsphone",
+                call_id: "",
+                keys: [
+                    {
+                        index: 0,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue("@bob:example.org"),
+        } as unknown as MatrixEvent);
+
+        const bobKeys = sess.getKeysForParticipant("@bob:example.org", "bobsphone")!;
+        expect(bobKeys).toHaveLength(1);
+        expect(bobKeys[0]).toEqual(Buffer.from("this is the key", "utf-8"));
+    });
+
+    it("collects keys at non-zero indices", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: "bobsphone",
+                call_id: "",
+                keys: [
+                    {
+                        index: 4,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue("@bob:example.org"),
+        } as unknown as MatrixEvent);
+
+        const bobKeys = sess.getKeysForParticipant("@bob:example.org", "bobsphone")!;
+        expect(bobKeys).toHaveLength(5);
+        expect(bobKeys[0]).toBeFalsy();
+        expect(bobKeys[1]).toBeFalsy();
+        expect(bobKeys[2]).toBeFalsy();
+        expect(bobKeys[3]).toBeFalsy();
+        expect(bobKeys[4]).toEqual(Buffer.from("this is the key", "utf-8"));
+    });
+
+    it("ignores keys event for the local participant", () => {
+        const mockRoom = makeMockRoom([membershipTemplate]);
+        sess = MatrixRTCSession.roomSessionForRoom(client, mockRoom);
+        sess.onCallEncryption({
+            getType: jest.fn().mockReturnValue("io.element.call.encryption_keys"),
+            getContent: jest.fn().mockReturnValue({
+                device_id: client.getDeviceId(),
+                call_id: "",
+                keys: [
+                    {
+                        index: 4,
+                        key: "dGhpcyBpcyB0aGUga2V5",
+                    },
+                ],
+            }),
+            getSender: jest.fn().mockReturnValue(client.getUserId()),
+        } as unknown as MatrixEvent);
+
+        const myKeys = sess.getKeysForParticipant(client.getUserId()!, client.getDeviceId()!)!;
+        expect(myKeys).toBeFalsy();
+    });
+});

spec/unit/matrixrtc/MatrixRTCSessionManager.spec.ts

@@ -0,0 +1,111 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import {
+    ClientEvent,
+    EventTimeline,
+    EventType,
+    IRoomTimelineData,
+    MatrixClient,
+    MatrixEvent,
+    RoomEvent,
+} from "../../../src";
+import { RoomStateEvent } from "../../../src/models/room-state";
+import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
+import { MatrixRTCSessionManagerEvents } from "../../../src/matrixrtc/MatrixRTCSessionManager";
+import { makeMockRoom } from "./mocks";
+
+const membershipTemplate: CallMembershipData = {
+    call_id: "",
+    scope: "m.room",
+    application: "m.call",
+    device_id: "AAAAAAA",
+    expires: 60 * 60 * 1000,
+    membershipID: "bloop",
+};
+
+describe("MatrixRTCSessionManager", () => {
+    let client: MatrixClient;
+
+    beforeEach(async () => {
+        client = new MatrixClient({ baseUrl: "base_url" });
+        client.matrixRTC.start();
+    });
+
+    afterEach(() => {
+        client.stopClient();
+        client.matrixRTC.stop();
+    });
+
+    it("Fires event when session starts", () => {
+        const onStarted = jest.fn();
+        client.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionStarted, onStarted);
+
+        try {
+            const room1 = makeMockRoom([membershipTemplate]);
+            jest.spyOn(client, "getRooms").mockReturnValue([room1]);
+
+            client.emit(ClientEvent.Room, room1);
+            expect(onStarted).toHaveBeenCalledWith(room1.roomId, client.matrixRTC.getActiveRoomSession(room1));
+        } finally {
+            client.matrixRTC.off(MatrixRTCSessionManagerEvents.SessionStarted, onStarted);
+        }
+    });
+
+    it("Fires event when session ends", () => {
+        const onEnded = jest.fn();
+        client.matrixRTC.on(MatrixRTCSessionManagerEvents.SessionEnded, onEnded);
+
+        const memberships = [membershipTemplate];
+
+        const room1 = makeMockRoom(memberships);
+        jest.spyOn(client, "getRooms").mockReturnValue([room1]);
+        jest.spyOn(client, "getRoom").mockReturnValue(room1);
+
+        client.emit(ClientEvent.Room, room1);
+
+        memberships.splice(0, 1);
+
+        const roomState = room1.getLiveTimeline().getState(EventTimeline.FORWARDS)!;
+        const membEvent = roomState.getStateEvents("")[0];
+
+        client.emit(RoomStateEvent.Events, membEvent, roomState, null);
+
+        expect(onEnded).toHaveBeenCalledWith(room1.roomId, client.matrixRTC.getActiveRoomSession(room1));
+    });
+
+    it("Calls onCallEncryption on encryption keys event", () => {
+        const room1 = makeMockRoom([membershipTemplate]);
+        jest.spyOn(client, "getRooms").mockReturnValue([room1]);
+        jest.spyOn(client, "getRoom").mockReturnValue(room1);
+
+        client.emit(ClientEvent.Room, room1);
+        const onCallEncryptionMock = jest.fn();
+        client.matrixRTC.getRoomSession(room1).onCallEncryption = onCallEncryptionMock;
+
+        const timelineEvent = {
+            getType: jest.fn().mockReturnValue(EventType.CallEncryptionKeysPrefix),
+            getContent: jest.fn().mockReturnValue({}),
+            getSender: jest.fn().mockReturnValue("@mock:user.example"),
+            getRoomId: jest.fn().mockReturnValue("!room:id"),
+            sender: {
+                userId: "@mock:user.example",
+            },
+        } as unknown as MatrixEvent;
+        client.emit(RoomEvent.Timeline, timelineEvent, undefined, undefined, false, {} as IRoomTimelineData);
+        expect(onCallEncryptionMock).toHaveBeenCalled();
+    });
+});

spec/unit/matrixrtc/mocks.ts

@@ -0,0 +1,57 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventType, MatrixEvent, Room } from "../../../src";
+import { CallMembershipData } from "../../../src/matrixrtc/CallMembership";
+import { randomString } from "../../../src/randomstring";
+
+export function makeMockRoom(memberships: CallMembershipData[], localAge: number | null = null): Room {
+    const roomId = randomString(8);
+    // Caching roomState here so it does not get recreated when calling `getLiveTimeline.getState()`
+    const roomState = makeMockRoomState(memberships, roomId, localAge);
+    return {
+        roomId: roomId,
+        getLiveTimeline: jest.fn().mockReturnValue({
+            getState: jest.fn().mockReturnValue(roomState),
+        }),
+    } as unknown as Room;
+}
+
+export function makeMockRoomState(memberships: CallMembershipData[], roomId: string, localAge: number | null = null) {
+    const event = mockRTCEvent(memberships, roomId, localAge);
+    return {
+        getStateEvents: (_: string, stateKey: string) => {
+            if (stateKey !== undefined) return event;
+            return [event];
+        },
+    };
+}
+
+export function mockRTCEvent(memberships: CallMembershipData[], roomId: string, localAge: number | null): MatrixEvent {
+    return {
+        getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix),
+        getContent: jest.fn().mockReturnValue({
+            memberships: memberships,
+        }),
+        getSender: jest.fn().mockReturnValue("@mock:user.example"),
+        getTs: jest.fn().mockReturnValue(1000),
+        localTimestamp: Date.now() - (localAge ?? 10),
+        getRoomId: jest.fn().mockReturnValue(roomId),
+        sender: {
+            userId: "@mock:user.example",
+        },
+    } as unknown as MatrixEvent;
+}

spec/unit/models/event.spec.ts

@@ -14,10 +14,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+import { MockedObject } from "jest-mock";
+
 import { MatrixEvent, MatrixEventEvent } from "../../../src/models/event";
 import { emitPromise } from "../../test-utils/test-utils";
 import { Crypto, IEventDecryptionResult } from "../../../src/crypto";
-import { IAnnotatedPushRule, PushRuleActionName, TweakName } from "../../../src";
+import {
+    IAnnotatedPushRule,
+    MatrixClient,
+    PushRuleActionName,
+    Room,
+    THREAD_RELATION_TYPE,
+    TweakName,
+} from "../../../src";
 
 describe("MatrixEvent", () => {
     it("should create copies of itself", () => {
@@ -61,31 +70,264 @@ describe("MatrixEvent", () => {
         expect(a.toSnapshot().isEquivalentTo(b)).toBe(false);
     });
 
-    it("should prune clearEvent when being redacted", () => {
-        const ev = new MatrixEvent({
-            type: "m.room.message",
-            content: {
-                body: "Test",
-            },
-            event_id: "$event1:server",
+    describe("redaction", () => {
+        it("should prune clearEvent when being redacted", () => {
+            const ev = createEvent("$event1:server", "Test");
+
+            expect(ev.getContent().body).toBe("Test");
+            expect(ev.getWireContent().body).toBe("Test");
+            ev.makeEncrypted("m.room.encrypted", { ciphertext: "xyz" }, "", "");
+            expect(ev.getContent().body).toBe("Test");
+            expect(ev.getWireContent().body).toBeUndefined();
+            expect(ev.getWireContent().ciphertext).toBe("xyz");
+
+            const mockClient = {} as unknown as MockedObject<MatrixClient>;
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const redaction = createRedaction(ev.getId()!);
+
+            ev.makeRedacted(redaction, room);
+            expect(ev.getContent().body).toBeUndefined();
+            expect(ev.getWireContent().body).toBeUndefined();
+            expect(ev.getWireContent().ciphertext).toBeUndefined();
         });
 
-        expect(ev.getContent().body).toBe("Test");
-        expect(ev.getWireContent().body).toBe("Test");
-        ev.makeEncrypted("m.room.encrypted", { ciphertext: "xyz" }, "", "");
-        expect(ev.getContent().body).toBe("Test");
-        expect(ev.getWireContent().body).toBeUndefined();
-        expect(ev.getWireContent().ciphertext).toBe("xyz");
+        it("should remain in the main timeline when redacted", async () => {
+            // Given an event in the main timeline
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const ev = createEvent("$event1:server");
 
-        const redaction = new MatrixEvent({
-            type: "m.room.redaction",
-            redacts: ev.getId(),
+            await room.addLiveEvents([ev]);
+            await room.createThreadsTimelineSets();
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([ev.getId()]);
+
+            // When I redact it
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then it remains in the main timeline
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([ev.getId()]);
         });
 
-        ev.makeRedacted(redaction);
-        expect(ev.getContent().body).toBeUndefined();
-        expect(ev.getWireContent().body).toBeUndefined();
-        expect(ev.getWireContent().ciphertext).toBeUndefined();
+        it("should keep thread roots in both timelines when redacted", async () => {
+            // Given a thread exists
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev]);
+            await room.createThreadsTimelineSets();
+            expect(threadRoot.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+
+            // When I redact the thread root
+            const redaction = createRedaction(ev.getId()!);
+            threadRoot.makeRedacted(redaction, room);
+
+            // Then it remains in the main timeline and the thread
+            expect(threadRoot.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+        });
+
+        it("should move into the main timeline when redacted", async () => {
+            // Given an event in a thread
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev]);
+            await room.createThreadsTimelineSets();
+            expect(ev.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId()]);
+
+            // When I redact it
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then it disappears from the thread and appears in the main timeline
+            expect(ev.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(ev.getId());
+        });
+
+        it("should move reactions to a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const reaction = createReactionEvent("$reaction:server", ev.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, reaction]);
+            await room.createThreadsTimelineSets();
+            expect(reaction.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId(), reaction.getId()]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the reaction moves into the main timeline
+            expect(reaction.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId(), reaction.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(reaction.getId());
+        });
+
+        it("should move edits of a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const edit = createEditEvent("$edit:server", ev.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, edit]);
+            await room.createThreadsTimelineSets();
+            expect(edit.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([threadRoot.getId(), ev.getId(), edit.getId()]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the edit moves into the main timeline
+            expect(edit.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId(), ev.getId(), edit.getId()]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(edit.getId());
+        });
+
+        it("should move reactions to replies to replies a redacted event into the main timeline", async () => {
+            // Given an event in a thread with a reaction
+            const mockClient = createMockClient();
+            const room = new Room("!roomid:e.xyz", mockClient, "myname");
+            const threadRoot = createEvent("$threadroot:server");
+            const ev = createThreadedEvent("$event1:server", threadRoot.getId()!);
+            const reply1 = createReplyEvent("$reply1:server", ev.getId()!);
+            const reply2 = createReplyEvent("$reply2:server", reply1.getId()!);
+            const reaction = createReactionEvent("$reaction:server", reply2.getId()!);
+
+            await room.addLiveEvents([threadRoot, ev, reply1, reply2, reaction]);
+            await room.createThreadsTimelineSets();
+            expect(reaction.threadRootId).toEqual(threadRoot.getId());
+            expect(mainTimelineLiveEventIds(room)).toEqual([threadRoot.getId()]);
+            expect(threadLiveEventIds(room, 0)).toEqual([
+                threadRoot.getId(),
+                ev.getId(),
+                reply1.getId(),
+                reply2.getId(),
+                reaction.getId(),
+            ]);
+
+            // When I redact the event
+            const redaction = createRedaction(ev.getId()!);
+            ev.makeRedacted(redaction, room);
+
+            // Then the replies move to the main thread and the reaction disappears
+            expect(reaction.threadRootId).toBeUndefined();
+            expect(mainTimelineLiveEventIds(room)).toEqual([
+                threadRoot.getId(),
+                ev.getId(),
+                reply1.getId(),
+                reply2.getId(),
+                reaction.getId(),
+            ]);
+            expect(threadLiveEventIds(room, 0)).not.toContain(reply1.getId());
+            expect(threadLiveEventIds(room, 0)).not.toContain(reply2.getId());
+            expect(threadLiveEventIds(room, 0)).not.toContain(reaction.getId());
+        });
+
+        function createMockClient(): MatrixClient {
+            return {
+                supportsThreads: jest.fn().mockReturnValue(true),
+                decryptEventIfNeeded: jest.fn().mockReturnThis(),
+                getUserId: jest.fn().mockReturnValue("@user:server"),
+            } as unknown as MockedObject<MatrixClient>;
+        }
+
+        function createEvent(eventId: string, body?: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    body: body ?? eventId,
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createThreadedEvent(eventId: string, threadRootId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "body": eventId,
+                    "m.relates_to": {
+                        rel_type: THREAD_RELATION_TYPE.name,
+                        event_id: threadRootId,
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createEditEvent(eventId: string, repliedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "body": "Edited",
+                    "m.new_content": {
+                        body: "Edited",
+                    },
+                    "m.relates_to": {
+                        event_id: repliedToId,
+                        rel_type: "m.replace",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createReplyEvent(eventId: string, repliedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.message",
+                content: {
+                    "m.relates_to": {
+                        event_id: repliedToId,
+                        key: "x",
+                        rel_type: "m.in_reply_to",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createReactionEvent(eventId: string, reactedToId: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.reaction",
+                content: {
+                    "m.relates_to": {
+                        event_id: reactedToId,
+                        key: "x",
+                        rel_type: "m.annotation",
+                    },
+                },
+                event_id: eventId,
+            });
+        }
+
+        function createRedaction(redactedEventid: string): MatrixEvent {
+            return new MatrixEvent({
+                type: "m.room.redaction",
+                redacts: redactedEventid,
+            });
+        }
     });
 
     describe("applyVisibilityEvent", () => {
@@ -308,4 +550,41 @@ describe("MatrixEvent", () => {
             });
         });
     });
+
+    it("should ignore thread relation on state events", async () => {
+        const stateEvent = new MatrixEvent({
+            event_id: "$event_id",
+            type: "some_state_event",
+            content: {
+                "foo": "bar",
+                "m.relates_to": {
+                    "event_id": "$thread_id",
+                    "m.in_reply_to": {
+                        event_id: "$thread_id",
+                    },
+                    "rel_type": "m.thread",
+                },
+            },
+            state_key: "",
+        });
+
+        expect(stateEvent.isState()).toBeTruthy();
+        expect(stateEvent.threadRootId).toBeUndefined();
+    });
 });
+
+function mainTimelineLiveEventIds(room: Room): Array<string> {
+    return room
+        .getLiveTimeline()
+        .getEvents()
+        .map((e) => e.getId()!);
+}
+
+function threadLiveEventIds(room: Room, threadIndex: number): Array<string> {
+    return room
+        .getThreads()
+        [threadIndex].getUnfilteredTimelineSet()
+        .getLiveTimeline()
+        .getEvents()
+        .map((e) => e.getId()!);
+}

spec/unit/models/room-receipts.spec.ts

@@ -0,0 +1,541 @@
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { FeatureSupport, MatrixClient, MatrixEvent, ReceiptContent, THREAD_RELATION_TYPE, Thread } from "../../../src";
+import { Room } from "../../../src/models/room";
+
+/**
+ * Note, these tests check the functionality of the RoomReceipts class, but most
+ * of them access that functionality via the surrounding Room class, because a
+ * room is required for RoomReceipts to function, and this matches the pattern
+ * of how this code is used in the wild.
+ */
+describe("RoomReceipts", () => {
+    beforeAll(() => {
+        jest.replaceProperty(Thread, "hasServerSideSupport", FeatureSupport.Stable);
+    });
+
+    afterAll(() => {
+        jest.restoreAllMocks();
+    });
+
+    it("reports events unread if there are no receipts", () => {
+        // Given there are no receipts in the room
+        const room = createRoom();
+        const [event] = createEvent();
+        room.addLiveEvents([event]);
+
+        // When I ask about any event, then it is unread
+        expect(room.hasUserReadEvent(readerId, event.getId()!)).toBe(false);
+    });
+
+    it("reports events we sent as read even if there are no (real) receipts", () => {
+        // Given there are no receipts in the room
+        const room = createRoom();
+        const [event] = createEventSentBy(readerId);
+        room.addLiveEvents([event]);
+
+        // When I ask about an event I sent, it is read (because a synthetic
+        // receipt was created and stored in RoomReceipts)
+        expect(room.hasUserReadEvent(readerId, event.getId()!)).toBe(true);
+    });
+
+    it("reports read if we receive an unthreaded receipt for this event", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event+user
+        room.addReceipt(createReceipt(readerId, event));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive an unthreaded receipt for a later event", () => {
+        // Given we have 2 events
+        const room = createRoom();
+        const [event1, event1Id] = createEvent();
+        const [event2] = createEvent();
+        room.addLiveEvents([event1, event2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createReceipt(readerId, event2));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+    });
+
+    it("reports read for a non-live event if we receive an unthreaded receipt for a live one", () => {
+        // Given we have 2 events: one live and one old
+        const room = createRoom();
+        const [oldEvent, oldEventId] = createEvent();
+        const [liveEvent] = createEvent();
+        room.addLiveEvents([liveEvent]);
+        createOldTimeline(room, [oldEvent]);
+
+        // When we receive a receipt for the live event
+        room.addReceipt(createReceipt(readerId, liveEvent));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, oldEventId)).toBe(true);
+    });
+
+    it("compares by timestamp if two events are in separate old timelines", () => {
+        // Given we have 2 events, both in old timelines, with event2 after
+        // event1 in terms of timestamps
+        const room = createRoom();
+        const [event1, event1Id] = createEvent();
+        const [event2, event2Id] = createEvent();
+        event1.event.origin_server_ts = 1;
+        event2.event.origin_server_ts = 2;
+        createOldTimeline(room, [event1]);
+        createOldTimeline(room, [event2]);
+
+        // When we receive a receipt for the older event
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then the earlier one is read and the later one is not
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an unthreaded receipt for an earlier event", () => {
+        // Given we have 2 events
+        const room = createRoom();
+        const [event1] = createEvent();
+        const [event2, event2Id] = createEvent();
+        room.addLiveEvents([event1, event2]);
+
+        // When we receive a receipt for the earlier event
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then the later one is unread
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an unthreaded receipt for a different user", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for another user
+        room.addReceipt(createReceipt(otherUserId, event));
+
+        // Then the event is still unread since the receipt was not for us
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // But it's read for the other person
+        expect(room.hasUserReadEvent(otherUserId, eventId)).toBe(true);
+    });
+
+    it("reports events we sent as read even if an earlier receipt arrives", () => {
+        // Given we sent an event after some other event
+        const room = createRoom();
+        const [previousEvent] = createEvent();
+        const [myEvent] = createEventSentBy(readerId);
+        room.addLiveEvents([previousEvent, myEvent]);
+
+        // And I just received a receipt for the previous event
+        room.addReceipt(createReceipt(readerId, previousEvent));
+
+        // When I ask about the event I sent, it is read (because of synthetic receipts)
+        expect(room.hasUserReadEvent(readerId, myEvent.getId()!)).toBe(true);
+    });
+
+    it("considers events after ones we sent to be unread", () => {
+        // Given we sent an event, then another event came in
+        const room = createRoom();
+        const [myEvent] = createEventSentBy(readerId);
+        const [laterEvent] = createEvent();
+        room.addLiveEvents([myEvent, laterEvent]);
+
+        // When I ask about the later event, it is unread (because it's after the synthetic receipt)
+        expect(room.hasUserReadEvent(readerId, laterEvent.getId()!)).toBe(false);
+    });
+
+    it("correctly reports readness even when receipts arrive out of order", () => {
+        // Given we have 3 events
+        const room = createRoom();
+        const [event1] = createEvent();
+        const [event2, event2Id] = createEvent();
+        const [event3, event3Id] = createEvent();
+        room.addLiveEvents([event1, event2, event3]);
+
+        // When we receive receipts for the older events out of order
+        room.addReceipt(createReceipt(readerId, event2));
+        room.addReceipt(createReceipt(readerId, event1));
+
+        // Then we correctly ignore the older receipt
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event3Id)).toBe(false);
+    });
+
+    it("reports read if we receive a threaded receipt for this event (main)", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [event, eventId] = createEvent();
+        room.addLiveEvents([event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event+user
+        room.addReceipt(createThreadedReceipt(readerId, event, "main"));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive a threaded receipt for this event (non-main)", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event, eventId] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for this event on this thread
+        room.addReceipt(createThreadedReceipt(readerId, event, rootId));
+
+        // Then that event is read
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+    });
+
+    it("reports read if we receive an threaded receipt for a later event", () => {
+        // Given we have 2 events in a thread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1, event1Id] = createThreadedEvent(root);
+        const [event2] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createThreadedReceipt(readerId, event2, rootId));
+
+        // Then the earlier one is read
+        expect(room.hasUserReadEvent(readerId, event1Id)).toBe(true);
+    });
+
+    it("reports unread if we receive an threaded receipt for an earlier event", () => {
+        // Given we have 2 events in a thread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1] = createThreadedEvent(root);
+        const [event2, event2Id] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2]);
+
+        // When we receive a receipt for the earlier event
+        room.addReceipt(createThreadedReceipt(readerId, event1, rootId));
+
+        // Then the later one is unread
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(false);
+    });
+
+    it("reports unread if we receive an threaded receipt for a different user", () => {
+        // Given my event exists and is unread
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event, eventId] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event]);
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // When we receive a receipt for another user
+        room.addReceipt(createThreadedReceipt(otherUserId, event, rootId));
+
+        // Then the event is still unread since the receipt was not for us
+        expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+        // But it's read for the other person
+        expect(room.hasUserReadEvent(otherUserId, eventId)).toBe(true);
+    });
+
+    it("reports unread if we receive a receipt for a later event in a different thread", () => {
+        // Given 2 events exist in different threads
+        const room = createRoom();
+        const [root1] = createEvent();
+        const [root2] = createEvent();
+        const [thread1, thread1Id] = createThreadedEvent(root1);
+        const [thread2] = createThreadedEvent(root2);
+        setupThread(room, root1);
+        setupThread(room, root2);
+        room.addLiveEvents([root1, root2, thread1, thread2]);
+
+        // When we receive a receipt for the later event
+        room.addReceipt(createThreadedReceipt(readerId, thread2, root2.getId()!));
+
+        // Then the old one is still unread since the receipt was not for this thread
+        expect(room.hasUserReadEvent(readerId, thread1Id)).toBe(false);
+    });
+
+    it("correctly reports readness even when threaded receipts arrive out of order", () => {
+        // Given we have 3 events
+        const room = createRoom();
+        const [root, rootId] = createEvent();
+        const [event1] = createThreadedEvent(root);
+        const [event2, event2Id] = createThreadedEvent(root);
+        const [event3, event3Id] = createThreadedEvent(root);
+        setupThread(room, root);
+        room.addLiveEvents([root, event1, event2, event3]);
+
+        // When we receive receipts for the older events out of order
+        room.addReceipt(createThreadedReceipt(readerId, event2, rootId));
+        room.addReceipt(createThreadedReceipt(readerId, event1, rootId));
+
+        // Then we correctly ignore the older receipt
+        expect(room.hasUserReadEvent(readerId, event2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, event3Id)).toBe(false);
+    });
+
+    it("correctly reports readness when mixing threaded and unthreaded receipts", () => {
+        // Given we have a setup from this presentation:
+        // https://docs.google.com/presentation/d/1H1gxRmRFAm8d71hCILWmpOYezsvdlb7cB6ANl-20Gns/edit?usp=sharing
+        //
+        //                     Main1----\
+        //                       |       ---Thread1a <- threaded receipt
+        //                       |             |
+        //                       |          Thread1b
+        // threaded receipt -> Main2--\
+        //                       |     ----------------Thread2a <- unthreaded receipt
+        //                     Main3                      |
+        //                                             Thread2b <- threaded receipt
+        //
+        const room = createRoom();
+        const [main1, main1Id] = createEvent();
+        const [main2, main2Id] = createEvent();
+        const [main3, main3Id] = createEvent();
+        const [thread1a, thread1aId] = createThreadedEvent(main1);
+        const [thread1b, thread1bId] = createThreadedEvent(main1);
+        const [thread2a, thread2aId] = createThreadedEvent(main2);
+        const [thread2b, thread2bId] = createThreadedEvent(main2);
+        setupThread(room, main1);
+        setupThread(room, main2);
+        room.addLiveEvents([main1, thread1a, thread1b, main2, thread2a, main3, thread2b]);
+
+        // And the timestamps on the events are consistent with the order above
+        main1.event.origin_server_ts = 1;
+        thread1a.event.origin_server_ts = 2;
+        thread1b.event.origin_server_ts = 3;
+        main2.event.origin_server_ts = 4;
+        thread2a.event.origin_server_ts = 5;
+        main3.event.origin_server_ts = 6;
+        thread2b.event.origin_server_ts = 7;
+        // (Note: in principle, we have the information needed to order these
+        // events without using their timestamps, since they all came in via
+        // addLiveEvents. In reality, some of them would have come in via the
+        // /relations API, making it impossible to get the correct ordering
+        // without MSC4033, which is why we fall back to timestamps. I.e. we
+        // definitely could fix the code to make the above
+        // timestamp-manipulation unnecessary, but it would only make this test
+        // neater, not actually help in the real world.)
+
+        // When the receipts arrive
+        room.addReceipt(createThreadedReceipt(readerId, main2, "main"));
+        room.addReceipt(createThreadedReceipt(readerId, thread1a, main1Id));
+        room.addReceipt(createReceipt(readerId, thread2a));
+        room.addReceipt(createThreadedReceipt(readerId, thread2b, main2Id));
+
+        // Then we correctly identify that only main3 is unread
+        expect(room.hasUserReadEvent(readerId, main1Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, main2Id)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, main3Id)).toBe(false);
+        expect(room.hasUserReadEvent(readerId, thread1aId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread1bId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread2aId)).toBe(true);
+        expect(room.hasUserReadEvent(readerId, thread2bId)).toBe(true);
+    });
+
+    describe("dangling receipts", () => {
+        it("reports unread if the unthreaded receipt is in a dangling state", () => {
+            const room = createRoom();
+            const [event, eventId] = createEvent();
+            // When we receive a receipt for this event+user
+            room.addReceipt(createReceipt(readerId, event));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the event to the room
+            // The receipt is removed from the dangling state
+            room.addLiveEvents([event]);
+
+            // Then the event is read
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+
+        it("reports unread if the threaded receipt is in a dangling state", () => {
+            const room = createRoom();
+            const [root, rootId] = createEvent();
+            const [event, eventId] = createThreadedEvent(root);
+            setupThread(room, root);
+
+            // When we receive a receipt for this event+user
+            room.addReceipt(createThreadedReceipt(readerId, event, rootId));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the events to the room
+            // The receipt is removed from the dangling state
+            room.addLiveEvents([root, event]);
+
+            // Then the event is read
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+
+        it("should handle multiple dangling receipts for the same event", () => {
+            const room = createRoom();
+            const [event, eventId] = createEvent();
+            // When we receive a receipt for this event+user
+            room.addReceipt(createReceipt(readerId, event));
+            // We receive another receipt in the same event for another user
+            room.addReceipt(createReceipt(otherUserId, event));
+
+            // The event is not added in the room
+            // So the receipt is in a dangling state
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(false);
+
+            // Add the event to the room
+            // The two receipts should be processed
+            room.addLiveEvents([event]);
+
+            // Then the event is read
+            // We expect that the receipt of `otherUserId` didn't replace/erase the receipt of `readerId`
+            expect(room.hasUserReadEvent(readerId, eventId)).toBe(true);
+        });
+    });
+});
+
+function createFakeClient(): MatrixClient {
+    return {
+        getUserId: jest.fn(),
+        getEventMapper: jest.fn().mockReturnValue(jest.fn()),
+        isInitialSyncComplete: jest.fn().mockReturnValue(true),
+        supportsThreads: jest.fn().mockReturnValue(true),
+        fetchRoomEvent: jest.fn().mockResolvedValue({}),
+        paginateEventTimeline: jest.fn(),
+        canSupport: { get: jest.fn() },
+    } as unknown as MatrixClient;
+}
+
+const senderId = "sender:s.ss";
+const readerId = "reader:r.rr";
+const otherUserId = "other:o.oo";
+
+function createRoom(): Room {
+    return new Room("!rid", createFakeClient(), "@u:s.nz", { timelineSupport: true });
+}
+
+let idCounter = 0;
+function nextId(): string {
+    return "$" + (idCounter++).toString(10);
+}
+
+/**
+ * Create an event and return it and its ID.
+ */
+function createEvent(): [MatrixEvent, string] {
+    return createEventSentBy(senderId);
+}
+
+/**
+ * Create an event with the supplied sender and return it and its ID.
+ */
+function createEventSentBy(customSenderId: string): [MatrixEvent, string] {
+    const event = new MatrixEvent({ sender: customSenderId, event_id: nextId() });
+    return [event, event.getId()!];
+}
+
+/**
+ * Create an event in the thread of the supplied root and return it and its ID.
+ */
+function createThreadedEvent(root: MatrixEvent): [MatrixEvent, string] {
+    const rootEventId = root.getId()!;
+    const event = new MatrixEvent({
+        sender: senderId,
+        event_id: nextId(),
+        content: {
+            "m.relates_to": {
+                event_id: rootEventId,
+                rel_type: THREAD_RELATION_TYPE.name,
+                ["m.in_reply_to"]: {
+                    event_id: rootEventId,
+                },
+            },
+        },
+    });
+    return [event, event.getId()!];
+}
+
+function createReceipt(userId: string, referencedEvent: MatrixEvent): MatrixEvent {
+    const content: ReceiptContent = {
+        [referencedEvent.getId()!]: {
+            "m.read": {
+                [userId]: {
+                    ts: 123,
+                },
+            },
+        },
+    };
+
+    return new MatrixEvent({
+        type: "m.receipt",
+        content,
+    });
+}
+
+function createThreadedReceipt(userId: string, referencedEvent: MatrixEvent, threadId: string): MatrixEvent {
+    const content: ReceiptContent = {
+        [referencedEvent.getId()!]: {
+            "m.read": {
+                [userId]: {
+                    ts: 123,
+                    thread_id: threadId,
+                },
+            },
+        },
+    };
+
+    return new MatrixEvent({
+        type: "m.receipt",
+        content,
+    });
+}
+
+/**
+ * Create a timeline in the timeline set that is not the live timeline.
+ */
+function createOldTimeline(room: Room, events: MatrixEvent[]) {
+    const oldTimeline = room.getUnfilteredTimelineSet().addTimeline();
+    room.getUnfilteredTimelineSet().addEventsToTimeline(events, true, oldTimeline);
+}
+
+/**
+ * Perform the hacks required for this room to create a thread based on the root
+ * event supplied.
+ */
+function setupThread(room: Room, root: MatrixEvent) {
+    const thread = room.createThread(root.getId()!, root, [root], false);
+    thread.initialEventsFetched = true;
+}

spec/unit/models/thread.spec.ts

@@ -18,8 +18,8 @@ import { mocked } from "jest-mock";
 
 import { MatrixClient, PendingEventOrdering } from "../../../src/client";
 import { Room, RoomEvent } from "../../../src/models/room";
-import { Thread, THREAD_RELATION_TYPE, ThreadEvent, FeatureSupport } from "../../../src/models/thread";
-import { makeThreadEvent, mkThread } from "../../test-utils/thread";
+import { FeatureSupport, Thread, THREAD_RELATION_TYPE, ThreadEvent } from "../../../src/models/thread";
+import { makeThreadEvent, mkThread, populateThread } from "../../test-utils/thread";
 import { TestClient } from "../../TestClient";
 import { emitPromise, mkEdit, mkMessage, mkReaction, mock } from "../../test-utils/test-utils";
 import { Direction, EventStatus, EventType, MatrixEvent } from "../../../src";
@@ -43,6 +43,7 @@ describe("Thread", () => {
         const myUserId = "@bob:example.org";
         const testClient = new TestClient(myUserId, "DEVICE", "ACCESS_TOKEN", undefined, { timelineSupport: false });
         const client = testClient.client;
+        client.supportsThreads = jest.fn().mockReturnValue(true);
         const room = new Room("123", client, myUserId, {
             pendingEventOrdering: PendingEventOrdering.Detached,
         });
@@ -148,20 +149,38 @@ describe("Thread", () => {
         });
 
         it("considers other events with no RR as unread", () => {
-            const { thread, events } = mkThread({
+            // Given a long thread exists
+            const { thread, events } = populateThread({
                 room,
                 client,
-                authorId: myUserId,
-                participantUserIds: [myUserId],
+                authorId: "@other:foo.com",
+                participantUserIds: ["@other:foo.com"],
                 length: 25,
                 ts: 190,
             });
 
-            // Before alice's last unthreaded receipt
-            expect(thread.hasUserReadEvent("@alice:example.org", events.at(1)!.getId() ?? "")).toBeTruthy();
+            const event1 = events.at(1)!;
+            const event2 = events.at(2)!;
+            const event24 = events.at(24)!;
 
-            // After alice's last unthreaded receipt
-            expect(thread.hasUserReadEvent("@alice:example.org", events.at(-1)!.getId() ?? "")).toBeFalsy();
+            // And we have read the second message in it with an unthreaded receipt
+            const receipt = new MatrixEvent({
+                type: "m.receipt",
+                room_id: room.roomId,
+                content: {
+                    // unthreaded receipt for the second message in the thread
+                    [event2.getId()!]: {
+                        [ReceiptType.Read]: {
+                            [myUserId]: { ts: 200 },
+                        },
+                    },
+                },
+            });
+            room.addReceipt(receipt);
+
+            // Then we have read the first message in the thread, and not the last
+            expect(thread.hasUserReadEvent(myUserId, event1.getId()!)).toBe(true);
+            expect(thread.hasUserReadEvent(myUserId, event24.getId()!)).toBe(false);
         });
 
         it("considers event as read if there's a more recent unthreaded receipt", () => {
@@ -300,6 +319,7 @@ describe("Thread", () => {
                 timelineSupport: false,
             });
             const client = testClient.client;
+            client.supportsThreads = jest.fn().mockReturnValue(true);
             const room = new Room("123", client, myUserId, {
                 pendingEventOrdering: PendingEventOrdering.Detached,
             });
@@ -354,6 +374,7 @@ describe("Thread", () => {
                 timelineSupport: false,
             });
             const client = testClient.client;
+            client.supportsThreads = jest.fn().mockReturnValue(true);
             const room = new Room("123", client, myUserId, {
                 pendingEventOrdering: PendingEventOrdering.Detached,
             });
@@ -405,6 +426,7 @@ describe("Thread", () => {
                 timelineSupport: false,
             });
             const client = testClient.client;
+            client.supportsThreads = jest.fn().mockReturnValue(true);
             const room = new Room("123", client, myUserId, {
                 pendingEventOrdering: PendingEventOrdering.Detached,
             });
@@ -477,13 +499,13 @@ describe("Thread", () => {
 
                 // And a thread with an added event (with later timestamp)
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 1, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 1, 100, 200, userId);
 
                 // Then a receipt was added to the thread
                 const receipt = thread.getReadReceiptForUserId(userId);
                 expect(receipt).toBeTruthy();
-                expect(receipt?.eventId).toEqual(message.getId());
-                expect(receipt?.data.ts).toEqual(100);
+                expect(receipt?.eventId).toEqual(message2.getId());
+                expect(receipt?.data.ts).toEqual(200);
                 expect(receipt?.data.thread_id).toEqual(thread.id);
 
                 // (And the receipt was synthetic)
@@ -501,14 +523,14 @@ describe("Thread", () => {
 
                 // And a thread with an added event with a lower timestamp than its other events
                 const userId = "user1";
-                const { thread } = await createThreadAndEvent(client, 200, 100, userId);
+                const { thread, message1 } = await createThreadAnd2Events(client, 300, 200, 100, userId);
 
-                // Then no receipt was added to the thread (the receipt is still
-                // for the thread root). This happens because since we have no
+                // Then the receipt is for the first message, because its
+                // timestamp is later. This happens because since we have no
                 // recursive relations support, we know that sometimes events
                 // appear out of order, so we have to check their timestamps as
                 // a guess of the correct order.
-                expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(thread.rootEvent?.getId());
+                expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(message1.getId());
             });
         });
 
@@ -526,11 +548,11 @@ describe("Thread", () => {
 
                 // And a thread with an added event (with later timestamp)
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 1, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 1, 100, 200, userId);
 
                 // Then a receipt was added to the thread
                 const receipt = thread.getReadReceiptForUserId(userId);
-                expect(receipt?.eventId).toEqual(message.getId());
+                expect(receipt?.eventId).toEqual(message2.getId());
             });
 
             it("Creates a local echo receipt even for events BEFORE an existing receipt", async () => {
@@ -546,22 +568,24 @@ describe("Thread", () => {
 
                 // And a thread with an added event with a lower timestamp than its other events
                 const userId = "user1";
-                const { thread, message } = await createThreadAndEvent(client, 200, 100, userId);
+                const { thread, message2 } = await createThreadAnd2Events(client, 300, 200, 100, userId);
 
-                // Then a receipt was added to the thread, because relations
-                // recursion is available, so we trust the server to have
-                // provided us with events in the right order.
+                // Then a receipt was added for the last message, even though it
+                // has lower ts, because relations recursion is available, so we
+                // trust the server to have provided us with events in the right
+                // order.
                 const receipt = thread.getReadReceiptForUserId(userId);
-                expect(receipt?.eventId).toEqual(message.getId());
+                expect(receipt?.eventId).toEqual(message2.getId());
             });
         });
 
-        async function createThreadAndEvent(
+        async function createThreadAnd2Events(
             client: MatrixClient,
             rootTs: number,
-            eventTs: number,
+            message1Ts: number,
+            message2Ts: number,
             userId: string,
-        ): Promise<{ thread: Thread; message: MatrixEvent }> {
+        ): Promise<{ thread: Thread; message1: MatrixEvent; message2: MatrixEvent }> {
             const room = new Room("room1", client, userId);
 
             // Given a thread
@@ -572,24 +596,41 @@ describe("Thread", () => {
                 participantUserIds: [],
                 ts: rootTs,
             });
-            // Sanity: the current receipt is for the thread root
-            expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(thread.rootEvent?.getId());
+            // Sanity: there is no read receipt on the thread yet because the
+            // thread events don't get properly added to the room by mkThread.
+            expect(thread.getReadReceiptForUserId(userId)).toBeNull();
 
             const awaitTimelineEvent = new Promise<void>((res) => thread.on(RoomEvent.Timeline, () => res()));
 
-            // When we add a message that is before the latest receipt
-            const message = makeThreadEvent({
+            // Add a message with ts message1Ts
+            const message1 = makeThreadEvent({
                 event: true,
                 rootEventId: thread.id,
                 replyToEventId: thread.id,
                 user: userId,
                 room: room.roomId,
-                ts: eventTs,
+                ts: message1Ts,
             });
-            await thread.addEvent(message, false, true);
+            await thread.addEvent(message1, false, true);
             await awaitTimelineEvent;
 
-            return { thread, message };
+            // Sanity: the thread now has a properly-added event, so this event
+            // has a synthetic receipt.
+            expect(thread.getReadReceiptForUserId(userId)?.eventId).toEqual(message1.getId());
+
+            // Add a message with ts message2Ts
+            const message2 = makeThreadEvent({
+                event: true,
+                rootEventId: thread.id,
+                replyToEventId: thread.id,
+                user: userId,
+                room: room.roomId,
+                ts: message2Ts,
+            });
+            await thread.addEvent(message2, false, true);
+            await awaitTimelineEvent;
+
+            return { thread, message1, message2 };
         }
 
         function createClientWithEventMapper(canSupport: Map<Feature, ServerSupport> = new Map()): MatrixClient {
@@ -671,6 +712,69 @@ describe("Thread", () => {
             });
         });
     });
+
+    describe("addEvent", () => {
+        describe("Given server support for threads", () => {
+            let previousThreadHasServerSideSupport: FeatureSupport;
+
+            beforeAll(() => {
+                previousThreadHasServerSideSupport = Thread.hasServerSideSupport;
+                Thread.hasServerSideSupport = FeatureSupport.Stable;
+            });
+
+            afterAll(() => {
+                Thread.hasServerSideSupport = previousThreadHasServerSideSupport;
+            });
+
+            it("Adds events even if they appear out of order", async () => {
+                // Given a thread exists
+                const client = createClient();
+                const user = "@alice:matrix.org";
+                const room = "!room:z";
+                const thread = await createThread(client, user, room);
+                const prevNumEvents = thread.timeline.length;
+
+                // When two messages come in but the later one has an older timestamp
+                const message1 = createThreadMessage(thread.id, user, room, "message1");
+                const message2 = createThreadMessage(thread.id, user, room, "message2");
+                message2.localTimestamp -= 10000;
+
+                await thread.addEvent(message1, false);
+                await thread.addEvent(message2, false);
+
+                // Then both events end up in the timeline
+                expect(thread.timeline.length - prevNumEvents).toEqual(2);
+                const lastEvent = thread.timeline.at(-1)!;
+                const secondLastEvent = thread.timeline.at(-2)!;
+                expect(lastEvent).toBe(message2);
+                expect(secondLastEvent).toBe(message1);
+            });
+
+            it("Adds events to start even if they appear out of order", async () => {
+                // Given a thread exists
+                const client = createClient();
+                const user = "@alice:matrix.org";
+                const room = "!room:z";
+                const thread = await createThread(client, user, room);
+                const prevNumEvents = thread.timeline.length;
+
+                // When two messages come in but the later one has an older timestamp
+                const message1 = createThreadMessage(thread.id, user, room, "message1");
+                const message2 = createThreadMessage(thread.id, user, room, "message2");
+                message2.localTimestamp -= 10000;
+
+                await thread.addEvent(message1, false);
+                await thread.addEvent(message2, true);
+
+                // Then both events end up in the timeline
+                expect(thread.timeline.length - prevNumEvents).toEqual(2);
+                const lastEvent = thread.timeline.at(-1)!;
+                const firstEvent = thread.timeline.at(0)!;
+                expect(lastEvent).toBe(message1);
+                expect(firstEvent).toBe(message2);
+            });
+        });
+    });
 });
 
 /**

spec/unit/notifications.spec.ts

@@ -106,6 +106,8 @@ describe("fixNotificationCountOnDecryption", () => {
             mockClient,
         );
 
+        room.addLiveEvents([event]);
+
         THREAD_ID = event.getId()!;
         threadEvent = mkEvent({
             type: EventType.RoomMessage,

spec/unit/oidc/authorize.spec.ts

@@ -0,0 +1,416 @@
+/**
+ * @jest-environment jsdom
+ */
+
+/*
+Copyright 2023 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import fetchMock from "fetch-mock-jest";
+import { mocked } from "jest-mock";
+import jwtDecode from "jwt-decode";
+
+import { Method } from "../../../src";
+import * as crypto from "../../../src/crypto/crypto";
+import { logger } from "../../../src/logger";
+import {
+    completeAuthorizationCodeGrant,
+    generateAuthorizationParams,
+    generateAuthorizationUrl,
+    generateOidcAuthorizationUrl,
+} from "../../../src/oidc/authorize";
+import { OidcError } from "../../../src/oidc/error";
+import { makeDelegatedAuthConfig, mockOpenIdConfiguration } from "../../test-utils/oidc";
+
+jest.mock("jwt-decode");
+
+// save for resetting mocks
+const realSubtleCrypto = crypto.subtleCrypto;
+
+describe("oidc authorization", () => {
+    const delegatedAuthConfig = makeDelegatedAuthConfig();
+    const authorizationEndpoint = delegatedAuthConfig.metadata.authorization_endpoint;
+    const tokenEndpoint = delegatedAuthConfig.metadata.token_endpoint;
+    const clientId = "xyz789";
+    const baseUrl = "https://test.com";
+
+    // 14.03.2022 16:15
+    const now = 1647270879403;
+
+    beforeAll(() => {
+        jest.spyOn(logger, "warn");
+        jest.setSystemTime(now);
+
+        fetchMock.get(delegatedAuthConfig.issuer + ".well-known/openid-configuration", mockOpenIdConfiguration());
+    });
+
+    afterEach(() => {
+        // @ts-ignore reset any ugly mocking we did
+        crypto.subtleCrypto = realSubtleCrypto;
+    });
+
+    it("should generate authorization params", () => {
+        const result = generateAuthorizationParams({ redirectUri: baseUrl });
+
+        expect(result.redirectUri).toEqual(baseUrl);
+
+        // random strings
+        expect(result.state.length).toEqual(8);
+        expect(result.nonce.length).toEqual(8);
+        expect(result.codeVerifier.length).toEqual(64);
+
+        const expectedScope =
+            "openid urn:matrix:org.matrix.msc2967.client:api:* urn:matrix:org.matrix.msc2967.client:device:";
+        expect(result.scope.startsWith(expectedScope)).toBeTruthy();
+        // deviceId of 10 characters is appended to the device scope
+        expect(result.scope.length).toEqual(expectedScope.length + 10);
+    });
+
+    describe("generateAuthorizationUrl()", () => {
+        it("should generate url with correct parameters", async () => {
+            // test the no crypto case here
+            // @ts-ignore mocking
+            crypto.subtleCrypto = undefined;
+
+            const authorizationParams = generateAuthorizationParams({ redirectUri: baseUrl });
+            const authUrl = new URL(
+                await generateAuthorizationUrl(authorizationEndpoint, clientId, authorizationParams),
+            );
+
+            expect(authUrl.searchParams.get("response_mode")).toEqual("query");
+            expect(authUrl.searchParams.get("response_type")).toEqual("code");
+            expect(authUrl.searchParams.get("client_id")).toEqual(clientId);
+            expect(authUrl.searchParams.get("code_challenge_method")).toEqual("S256");
+            expect(authUrl.searchParams.get("scope")).toEqual(authorizationParams.scope);
+            expect(authUrl.searchParams.get("state")).toEqual(authorizationParams.state);
+            expect(authUrl.searchParams.get("nonce")).toEqual(authorizationParams.nonce);
+
+            // crypto not available, plain text code_challenge is used
+            expect(authUrl.searchParams.get("code_challenge")).toEqual(authorizationParams.codeVerifier);
+            expect(logger.warn).toHaveBeenCalledWith(
+                "A secure context is required to generate code challenge. Using plain text code challenge",
+            );
+        });
+    });
+
+    describe("generateOidcAuthorizationUrl()", () => {
+        it("should generate url with correct parameters", async () => {
+            const nonce = "abc123";
+
+            const metadata = delegatedAuthConfig.metadata;
+
+            const authUrl = new URL(
+                await generateOidcAuthorizationUrl({
+                    metadata,
+                    homeserverUrl: baseUrl,
+                    clientId,
+                    redirectUri: baseUrl,
+                    nonce,
+                }),
+            );
+
+            expect(authUrl.searchParams.get("response_mode")).toEqual("query");
+            expect(authUrl.searchParams.get("response_type")).toEqual("code");
+            expect(authUrl.searchParams.get("client_id")).toEqual(clientId);
+            expect(authUrl.searchParams.get("code_challenge_method")).toEqual("S256");
+            // scope minus the 10char random device id at the end
+            expect(authUrl.searchParams.get("scope")!.slice(0, -10)).toEqual(
+                "openid urn:matrix:org.matrix.msc2967.client:api:* urn:matrix:org.matrix.msc2967.client:device:",
+            );
+            expect(authUrl.searchParams.get("state")).toBeTruthy();
+            expect(authUrl.searchParams.get("nonce")).toEqual(nonce);
+
+            expect(authUrl.searchParams.get("code_challenge")).toBeTruthy();
+        });
+
+        it("should generate url with create prompt", async () => {
+            const nonce = "abc123";
+
+            const metadata = delegatedAuthConfig.metadata;
+
+            const authUrl = new URL(
+                await generateOidcAuthorizationUrl({
+                    metadata,
+                    homeserverUrl: baseUrl,
+                    clientId,
+                    redirectUri: baseUrl,
+                    nonce,
+                    prompt: "create",
+                }),
+            );
+
+            expect(authUrl.searchParams.get("prompt")).toEqual("create");
+        });
+    });
+
+    describe("completeAuthorizationCodeGrant", () => {
+        const homeserverUrl = "https://server.org/";
+        const identityServerUrl = "https://id.org/";
+        const nonce = "test-nonce";
+        const redirectUri = baseUrl;
+        const code = "auth_code_xyz";
+        const validBearerTokenResponse = {
+            token_type: "Bearer",
+            access_token: "test_access_token",
+            refresh_token: "test_refresh_token",
+            id_token: "valid.id.token",
+            expires_in: 300,
+        };
+
+        const metadata = mockOpenIdConfiguration();
+
+        const validDecodedIdToken = {
+            // nonce matches
+            nonce,
+            // not expired
+            exp: Date.now() / 1000 + 100000,
+            // audience is this client
+            aud: clientId,
+            // issuer matches
+            iss: metadata.issuer,
+            sub: "123",
+        };
+
+        const mockSessionStorage = (state: Record<string, unknown>): void => {
+            jest.spyOn(sessionStorage.__proto__, "getItem").mockImplementation((key: unknown) => {
+                return state[key as string] ?? null;
+            });
+            jest.spyOn(sessionStorage.__proto__, "setItem").mockImplementation(
+                // @ts-ignore mock type
+                (key: string, value: unknown) => (state[key] = value),
+            );
+            jest.spyOn(sessionStorage.__proto__, "removeItem").mockImplementation((key: unknown) => {
+                const { [key as string]: value, ...newState } = state;
+                state = newState;
+                return value;
+            });
+        };
+
+        const getValueFromStorage = <T = string>(state: string, key: string): T => {
+            const storedState = window.sessionStorage.getItem(`mx_oidc_${state}`)!;
+            return JSON.parse(storedState)[key] as unknown as T;
+        };
+
+        /**
+         * These tests kind of integration test oidc auth, by using `generateOidcAuthorizationUrl` and mocked storage
+         * to mock the use case of initiating oidc auth, putting state in storage, redirecting to OP,
+         * then returning and using state to verfiy.
+         * Returns random state string used to access storage
+         * @param params
+         */
+        const setupState = async (params = {}): Promise<string> => {
+            const url = await generateOidcAuthorizationUrl({
+                metadata,
+                redirectUri,
+                clientId,
+                homeserverUrl,
+                identityServerUrl,
+                nonce,
+                ...params,
+            });
+
+            const state = new URL(url).searchParams.get("state")!;
+
+            // add the scope with correct deviceId to the mocked bearer token response
+            const scope = getValueFromStorage(state, "scope");
+            fetchMock.post(metadata.token_endpoint, {
+                status: 200,
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                ...validBearerTokenResponse,
+                scope,
+            });
+
+            return state;
+        };
+
+        beforeEach(() => {
+            fetchMock.mockClear();
+            fetchMock.resetBehavior();
+
+            fetchMock.get(`${metadata.issuer}.well-known/openid-configuration`, metadata);
+            fetchMock.get(`${metadata.issuer}jwks`, {
+                status: 200,
+                headers: {
+                    "Content-Type": "application/json",
+                },
+                keys: [],
+            });
+
+            mockSessionStorage({});
+
+            mocked(jwtDecode).mockReturnValue(validDecodedIdToken);
+        });
+
+        it("should make correct request to the token endpoint", async () => {
+            const state = await setupState();
+            const codeVerifier = getValueFromStorage(state, "code_verifier");
+            await completeAuthorizationCodeGrant(code, state);
+
+            expect(fetchMock).toHaveBeenCalledWith(
+                metadata.token_endpoint,
+                expect.objectContaining({
+                    method: Method.Post,
+                    credentials: "same-origin",
+                    headers: {
+                        "Accept": "application/json",
+                        "Content-Type": "application/x-www-form-urlencoded",
+                    },
+                }),
+            );
+
+            // check body is correctly formed
+            const queryParams = fetchMock.mock.calls.find(([endpoint]) => endpoint === metadata.token_endpoint)![1]!
+                .body as URLSearchParams;
+            expect(queryParams.get("grant_type")).toEqual("authorization_code");
+            expect(queryParams.get("client_id")).toEqual(clientId);
+            expect(queryParams.get("code_verifier")).toEqual(codeVerifier);
+            expect(queryParams.get("redirect_uri")).toEqual(redirectUri);
+            expect(queryParams.get("code")).toEqual(code);
+        });
+
+        it("should return with valid bearer token", async () => {
+            const state = await setupState();
+            const scope = getValueFromStorage(state, "scope");
+            const result = await completeAuthorizationCodeGrant(code, state);
+
+            expect(result).toEqual({
+                homeserverUrl,
+                identityServerUrl,
+                oidcClientSettings: {
+                    clientId,
+                    issuer: metadata.issuer,
+                },
+                tokenResponse: {
+                    access_token: validBearerTokenResponse.access_token,
+                    id_token: validBearerTokenResponse.id_token,
+                    refresh_token: validBearerTokenResponse.refresh_token,
+                    token_type: validBearerTokenResponse.token_type,
+                    // this value is slightly unstable because it uses the clock
+                    expires_at: result.tokenResponse.expires_at,
+                    scope,
+                },
+                idTokenClaims: result.idTokenClaims,
+            });
+        });
+
+        it("should return with valid bearer token where token_type is lowercase", async () => {
+            const state = await setupState();
+            const scope = getValueFromStorage(state, "scope");
+            const tokenResponse = {
+                ...validBearerTokenResponse,
+                scope,
+                token_type: "bearer",
+            };
+            fetchMock.post(
+                tokenEndpoint,
+                {
+                    headers: {
+                        "Content-Type": "application/json",
+                    },
+                    ...tokenResponse,
+                },
+                { overwriteRoutes: true },
+            );
+
+            const result = await completeAuthorizationCodeGrant(code, state);
+
+            expect(result).toEqual({
+                homeserverUrl,
+                identityServerUrl,
+                oidcClientSettings: {
+                    clientId,
+                    issuer: metadata.issuer,
+                },
+                // results in token that uses 'Bearer' token type
+                tokenResponse: {
+                    access_token: validBearerTokenResponse.access_token,
+                    id_token: validBearerTokenResponse.id_token,
+                    refresh_token: validBearerTokenResponse.refresh_token,
+                    token_type: "Bearer",
+                    // this value is slightly unstable because it uses the clock
+                    expires_at: result.tokenResponse.expires_at,
+                    scope,
+                },
+                idTokenClaims: result.idTokenClaims,
+            });
+
+            expect(result.tokenResponse.token_type).toEqual("Bearer");
+        });
+
+        it("should throw when state is not found in storage", async () => {
+            // don't setup sessionStorage with expected state
+            const state = "abc123";
+            fetchMock.post(
+                metadata.token_endpoint,
+                {
+                    status: 500,
+                },
+                { overwriteRoutes: true },
+            );
+            await expect(() => completeAuthorizationCodeGrant(code, state)).rejects.toThrow(
+                new Error(OidcError.MissingOrInvalidStoredState),
+            );
+        });
+
+        it("should throw with code exchange failed error when request fails", async () => {
+            const state = await setupState();
+            fetchMock.post(
+                metadata.token_endpoint,
+                {
+                    status: 500,
+                },
+                { overwriteRoutes: true },
+            );
+            await expect(() => completeAuthorizationCodeGrant(code, state)).rejects.toThrow(
+                new Error(OidcError.CodeExchangeFailed),
+            );
+        });
+
+        it("should throw invalid token error when token is invalid", async () => {
+            const state = await setupState();
+            const invalidBearerTokenResponse = {
+                ...validBearerTokenResponse,
+                access_token: null,
+            };
+            fetchMock.post(
+                metadata.token_endpoint,
+                {
+                    headers: {
+                        "Content-Type": "application/json",
+                    },
+                    ...invalidBearerTokenResponse,
+                },
+                { overwriteRoutes: true },
+            );
+            await expect(() => completeAuthorizationCodeGrant(code, state)).rejects.toThrow(
+                new Error(OidcError.InvalidBearerTokenResponse),
+            );
+        });
+
+        it("should throw invalid id token error when id_token is invalid", async () => {
+            const state = await setupState();
+            mocked(jwtDecode).mockReturnValue({
+                ...validDecodedIdToken,
+                // invalid audience
+                aud: "something-else",
+            });
+            await expect(() => completeAuthorizationCodeGrant(code, state)).rejects.toThrow(
+                new Error(OidcError.InvalidIdToken),
+            );
+        });
+    });
+});