Update mix packaging

The "bookmarks_to_pep" command exports the bookmarks of the specified
user from private XML storage to PEP.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,6 +1,6 @@
 Environment
 -----------
-- ejabberd version: 18.06
+- ejabberd version: 18.09
 - Erlang version: `erl +V`
 - OS: Linux (Debian)
 - Installed from: source | distro package | official deb/rpm | official binary installer | other

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+labels: 
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.travis.yml

@@ -1,9 +1,9 @@
 language: erlang
 
 otp_release:
-  - 17.5
-  - 18.3
-  - 19.2
+  - 19.0
+  - 20.3
+  - 21.1
 
 services:
   - redis-server
@@ -33,9 +33,12 @@ before_script:
   - mysql -u root -e "CREATE USER 'ejabberd_test'@'localhost' IDENTIFIED BY 'ejabberd_test';"
   - mysql -u root -e "CREATE DATABASE ejabberd_test;"
   - mysql -u root -e "GRANT ALL ON ejabberd_test.* TO 'ejabberd_test'@'localhost';"
+  - mysql -u root ejabberd_test < sql/mysql.sql
   - psql -U postgres -c "CREATE USER ejabberd_test WITH PASSWORD 'ejabberd_test';"
   - psql -U postgres -c "CREATE DATABASE ejabberd_test;"
+  - psql -U postgres ejabberd_test -f sql/pg.sql
   - psql -U postgres -c "GRANT ALL PRIVILEGES ON DATABASE ejabberd_test TO ejabberd_test;"
+  - psql -U postgres ejabberd_test -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO ejabberd_test;"
 
 script:
   - ./autogen.sh

CONTRIBUTING.md

@@ -0,0 +1,148 @@
+# Contributing to ejabberd
+
+We'd love for you to contribute to our source code and to make ejabberd even better than it is
+today! Here are the guidelines we'd like you to follow:
+
+* [Code of Conduct](#coc)
+* [Questions and Problems](#question)
+* [Issues and Bugs](#issue)
+* [Feature Requests](#feature)
+* [Issue Submission Guidelines](#submit)
+* [Pull Request Submission Guidelines](#submit-pr)
+* [Signing the CLA](#cla)
+
+## <a name="coc"></a> Code of Conduct
+
+Help us keep ejabberd community open-minded and inclusive. Please read and follow our [Code of Conduct][coc].
+
+## <a name="requests"></a> Questions, Bugs, Features
+
+### <a name="question"></a> Got a Question or Problem?
+
+Do not open issues for general support questions as we want to keep GitHub issues for bug reports
+and feature requests. You've got much better chances of getting your question answered on dedicated
+support platforms, the best being [Stack Overflow][stackoverflow].
+
+Stack Overflow is a much better place to ask questions since:
+
+- there are thousands of people willing to help on Stack Overflow
+- questions and answers stay available for public viewing so your question / answer might help
+  someone else
+- Stack Overflow's voting system assures that the best answers are prominently visible.
+
+To save your and our time, we will systematically close all issues that are requests for general
+support and redirect people to the section you are reading right now.
+
+Other channels for support are:
+- [ejabberd Mailing List][list]
+- [ejabberd XMPP room][muc]: ejabberd@conference.process-one.net
+
+### <a name="issue"></a> Found an Issue or Bug?
+
+If you find a bug in the source code, you can help us by submitting an issue to our
+[GitHub Repository][github]. Even better, you can submit a Pull Request with a fix.
+
+### <a name="feature"></a> Missing a Feature?
+
+You can request a new feature by submitting an issue to our [GitHub Repository][github-issues].
+
+If you would like to implement a new feature then consider what kind of change it is:
+
+* **Major Changes** that you wish to contribute to the project should be discussed first in an
+  [GitHub issue][github-issues] that clearly outlines the changes and benefits of the feature.
+* **Small Changes** can directly be crafted and submitted to the [GitHub Repository][github]
+  as a Pull Request. See the section about [Pull Request Submission Guidelines](#submit-pr).
+
+## <a name="submit"></a> Issue Submission Guidelines
+
+Before you submit your issue search the archive, maybe your question was already answered.
+
+If your issue appears to be a bug, and hasn't been reported, open a new issue. Help us to maximize
+the effort we can spend fixing issues and adding new features, by not reporting duplicate issues.
+
+The "[new issue][github-new-issue]" form contains a number of prompts that you should fill out to
+make it easier to understand and categorize the issue.
+
+## <a name="submit-pr"></a> Pull Request Submission Guidelines
+
+By submitting a pull request for a code or doc contribution, you need to have the right
+to grant your contribution's copyright license to ProcessOne. Please check [ProcessOne CLA][cla]
+for details.
+
+Before you submit your pull request consider the following guidelines:
+
+* Search [GitHub][github-pr] for an open or closed Pull Request
+  that relates to your submission. You don't want to duplicate effort.
+* Create the [development environment][developer-setup]
+* Make your changes in a new git branch:
+
+    ```shell
+    git checkout -b my-fix-branch master
+    ```
+* Test your changes and, if relevant, expand the automated test suite.
+* Create your patch commit, including appropriate test cases.
+* If the changes affect public APIs, change or add relevant [documentation][doc-repo].
+* Commit your changes using a descriptive commit message.
+
+    ```shell
+    git commit -a
+    ```
+  Note: the optional commit `-a` command line option will automatically "add" and "rm" edited files.
+
+* Push your branch to GitHub:
+
+    ```shell
+    git push origin my-fix-branch
+    ```
+
+* In GitHub, send a pull request to `ejabberd:master`. This will trigger the Travis integration and run the test.
+We will also notify you if you have not yet signed the [contribution agreement][cla].
+
+* If you find that the Travis integration has failed, look into the logs on Travis to find out
+if your changes caused test failures, the commit message was malformed etc. If you find that the
+tests failed or times out for unrelated reasons, you can ping a team member so that the build can be
+restarted.
+
+* If we suggest changes, then:
+
+  * Make the required updates.
+  * Test your changes and test cases.
+  * Commit your changes to your branch (e.g. `my-fix-branch`).
+  * Push the changes to your GitHub repository (this will update your Pull Request).
+
+    You can also amend the initial commits and force push them to the branch.
+
+    ```shell
+    git rebase master -i
+    git push origin my-fix-branch -f
+    ```
+
+    This is generally easier to follow, but separate commits are useful if the Pull Request contains
+    iterations that might be interesting to see side-by-side.
+
+That's it! Thank you for your contribution!
+
+## <a name="cla"></a> Signing the Contributor License Agreement (CLA)
+
+Upon submmitting a Pull Request, we will ask you to sign our CLA if you haven't done
+so before. It's a quick process, we promise, and you will be able to do it all online
+
+You can read [ProcessOne Contribution License Agreement][cla] in PDF.
+
+This is part of the legal framework of the open-source ecosystem that adds some red tape,
+but protects both the contributor and the company / foundation behind the project. It also
+gives us the option to relicense the code with a more permissive license in the future.
+
+
+[coc]: https://github.com/processone/ejabberd/blob/master/CODE_OF_CONDUCT.md
+[stackoverflow]: https://stackoverflow.com/questions/tagged/ejabberd?sort=newest
+[list]: http://lists.jabber.ru/mailman/listinfo/ejabberd
+[muc]: xmpp:ejabberd@conference.process-one.net
+[github]: https://github.com/processone/ejabberd
+[github-issues]: https://github.com/processone/ejabberd/issues
+[github-new-issue]: https://github.com/processone/ejabberd/issues/new
+[github-pr]: https://github.com/processone/ejabberd/pulls
+[doc-repo]: https://github.com/processone/docs.ejabberd.im
+[developer-setup]: https://docs.ejabberd.im/developer/
+[cla]: https://www.process-one.net/resources/ejabberd-cla.pdf
+[license]: https://github.com/processone/ejabberd/blob/master/COPYING

CONTRIBUTORS.md

@@ -0,0 +1,37 @@
+# Contributors
+
+We would like to thanks official ejabberd source code contributors:
+
+- Sergey Abramyan
+- Badlop
+- Ludovic Bocquet
+- Emilio Bustos
+- Thiago Camargo
+- Juan Pablo Carlino
+- Paweł Chmielowski
+- Gabriel Gatu
+- Tsukasa Hamano
+- Konstantinos Kallas
+- Evgeny Khramtsov
+- Ben Langfeld
+- Peter Lemenkov
+- Anna Mukharram
+- Johan Oudinet
+- Pablo Polvorin
+- Mickaël Rémond
+- Matthias Rieber
+- Rafael Roemhild
+- Christophe Romain
+- Jérôme Sautret
+- Sonny Scroggin
+- Alexey Shchepin
+- Shelley Shyan
+- Radoslaw Szymczyszyn
+- Stu Tomlinson
+- Christian Ulrich
+- Holger Weiß
+
+Please, if you think we are missing your contribution, do not hesitate to contact us at ProcessOne.
+In case you do not want to appear in this list, please, let us know as well.
+
+Thanks !

Makefile.in

@@ -152,7 +152,7 @@ DEPS_FILES_FILTERED=$(filter-out $(BINARIES) deps/elixir/ebin/elixir.app,$(DEPS_
 DEPS_DIRS=$(sort deps/ $(foreach DEP,$(DEPS),deps/$(DEP)/) $(dir $(DEPS_FILES)))
 
 MAIN_FILES=$(filter-out %/configure.beam,$(call FILES_WILDCARD,ebin/*.beam ebin/*.app priv/msgs/*.msg priv/css/*.css priv/img/*.png priv/js/*.js priv/lib/* include/*.hrl COPYING))
-MAIN_DIRS=$(sort $(dir $(MAIN_FILES)) priv/bin priv/sql)
+MAIN_DIRS=$(sort $(dir $(MAIN_FILES)) priv/bin priv/sql priv/lua)
 
 define DEP_VERSION_template
 DEP_$(1)_VERSION:=$(shell $(SED) -e '/vsn/!d;s/.*, *"/$(1)-/;s/".*//' $(2) 2>/dev/null)
@@ -184,7 +184,10 @@ $(call TO_DEST,priv/sql/lite.sql): sql/lite.sql $(call TO_DEST,priv/sql)
 $(call TO_DEST,priv/bin/captcha.sh): tools/captcha.sh $(call TO_DEST,priv/bin)
 	$(INSTALL) -m 755 $(O_USER) $< $@
 
-copy-files-sub2: $(call TO_DEST,$(DEPS_FILES) $(MAIN_FILES) priv/bin/captcha.sh priv/sql/lite.sql)
+$(call TO_DEST,priv/lua/redis_sm.lua): priv/lua/redis_sm.lua $(call TO_DEST,priv/lua)
+	$(INSTALL) -m 644 $< $@
+
+copy-files-sub2: $(call TO_DEST,$(DEPS_FILES) $(MAIN_FILES) priv/bin/captcha.sh priv/sql/lite.sql priv/lua/redis_sm.lua)
 
 .PHONY: $(call TO_DEST,$(DEPS_FILES) $(MAIN_DIRS) $(DEPS_DIRS))
 

README

@@ -1,174 +0,0 @@
-ejabberd Community Edition 
-==========================
-
-[![Build Status](https://travis-ci.org/processone/ejabberd.svg?branch=master)](https://travis-ci.org/processone/ejabberd) [![Hex version](https://img.shields.io/hexpm/v/ejabberd.svg "Hex version")](https://hex.pm/packages/ejabberd)
-
-ejabberd is a distributed, fault-tolerant technology that allows the creation
-of large-scale instant messaging applications. The server can reliably support
-thousands of simultaneous users on a single node and has been designed to
-provide exceptional standards of fault tolerance. As an open source
-technology, based on industry-standards, ejabberd can be used to build bespoke
-solutions very cost effectively.
-
-
-Key Features
-------------
-
-- **Cross-platform**  
-  ejabberd runs under Microsoft Windows and Unix-derived systems such as
-  Linux, FreeBSD and NetBSD.
-
-- **Distributed**  
-  You can run ejabberd on a cluster of machines and all of them will serve the
-  same XMPP domain(s). When you need more capacity you can simply add a new
-  cheap node to your cluster. Accordingly, you do not need to buy an expensive
-  high-end machine to support tens of thousands concurrent users.
-
-- **Fault-tolerant**  
-  You can deploy an ejabberd cluster so that all the information required for
-  a properly working service will be replicated permanently on all nodes. This
-  means that if one of the nodes crashes, the others will continue working
-  without disruption. In addition, nodes also can be added or replaced ‘on
-  the fly’.
-
-- **Administrator-friendly**  
-  ejabberd is built on top of the Open Source Erlang. As a result you do not
-  need to install an external database, an external web server, amongst others
-  because everything is already included, and ready to run out of the box.
-  Other administrator benefits include:
-  - Comprehensive documentation.
-  - Straightforward installers for Linux and Mac OS X.
-  - Web administration.
-  - Shared roster groups.
-  - Command line administration tool.
-  - Can integrate with existing authentication mechanisms.
-  - Capability to send announce messages.
-
-- **Internationalized**  
-  ejabberd leads in internationalization. Hence it is very well suited in a
-  globalized world. Related features are:
-  - Translated to 25 languages.
-  - Support for IDNA.
-
-- **Open Standards**  
-  ejabberd is the first Open Source Jabber server claiming to fully comply to
-  the XMPP standard.
-  - Fully XMPP-compliant.
-  - XML-based protocol.
-  - Many protocols supported.
-
-
-Additional Features
--------------------
-
-Moreover, ejabberd comes with a wide range of other state-of-the-art features:
-
-- **Modularity**
-  - Load only the modules you want.
-  - Extend ejabberd with your own custom modules.
-
-- **Security**
-  - SASL and STARTTLS for c2s and s2s connections.
-  - STARTTLS and Dialback s2s connections.
-  - Web Admin accessible via HTTPS secure access.
-
-- **Databases**
-  - Internal database for fast deployment (Mnesia).
-  - Native MySQL support.
-  - Native PostgreSQL support.
-  - ODBC data storage support.
-  - Microsoft SQL Server support.
-
-- **Authentication**
-  - Internal authentication.
-  - PAM, LDAP and ODBC.
-  - External authentication script.
-
-- **Others**
-  - Support for virtual hosting.
-  - Compressing XML streams with Stream Compression (XEP-0138).
-  - Statistics via Statistics Gathering (XEP-0039).
-  - IPv6 support both for c2s and s2s connections.
-  - Multi-User Chat module with support for clustering and HTML logging.
-  - Users Directory based on users vCards.
-  - Publish-Subscribe component with support for Personal Eventing.
-  - Support for web clients: HTTP Polling and HTTP Binding (BOSH).
-  - Component support: interface with networks such as AIM, ICQ and MSN.
-
-
-Quickstart guide
-----------------
-
-### 0. Requirements
-
-To compile ejabberd you need:
-
- - GNU Make.
- - GCC.
- - Libexpat 1.95 or higher.
- - Libyaml 0.1.4 or higher.
- - Erlang/OTP 17.5 or higher.
- - OpenSSL 1.0.0 or higher, for STARTTLS, SASL and SSL encryption.
- - Zlib 1.2.3 or higher, for Stream Compression support (XEP-0138). Optional.
- - PAM library. Optional. For Pluggable Authentication Modules (PAM).
- - ImageMagick's Convert program. Optional. For CAPTCHA challenges.
-
-If your system splits packages in libraries and development headers, you must
-install the development packages also.
-
-### 1. Compile and install on *nix systems
-
-To compile ejabberd, execute the following commands.  The first one is only
-necessary if your source tree didn't come with a `configure` script (In this
-case you need autoconf installed).
-
-    ./autogen.sh
-    ./configure
-    make
-
-To install ejabberd, run this command with system administrator rights (root
-user):
-
-    sudo make install
-
-These commands will:
-
-- Install the configuration files in `/etc/ejabberd/`
-- Install ejabberd binary, header and runtime files in `/lib/ejabberd/`
-- Install the administration script: `/sbin/ejabberdctl`
-- Install ejabberd documentation in `/share/doc/ejabberd/`
-- Create a spool directory: `/var/lib/ejabberd/`
-- Create a directory for log files: `/var/log/ejabberd/`
-
-
-### 2. Start ejabberd
-
-You can use the `ejabberdctl` command line administration script to
-start and stop ejabberd. For example:
-
-    ejabberdctl start
-
-
-For detailed information please refer to the ejabberd Installation and
-Operation Guide available online and in the `doc` directory of the source
-tarball.
-
-
-Development
------------
-
-In order to assist in the development of ejabberd, and particularly the
-execution of the test suite, a Vagrant environment is available at
-https://github.com/processone/ejabberd-vagrant-dev.
-
-To start ejabberd in development mode from the repository directory, you can
-type a command like:
-
-    EJABBERD_CONFIG_PATH=ejabberd.yml erl -pa ebin -pa deps/*/ebin -pa test -pa deps/elixir/lib/*/ebin/ -s ejabberd
-
-Links
------
-
-- Documentation: http://docs.ejabberd.im
-- Community site: https://www.ejabberd.im
-- ejabberd commercial offering and support: https://www.process-one.net/en/ejabberd

README.md

@@ -1 +0,0 @@
-README

README.md

@@ -0,0 +1,174 @@
+ejabberd Community Edition 
+==========================
+
+[![Build Status](https://travis-ci.org/processone/ejabberd.svg?branch=master)](https://travis-ci.org/processone/ejabberd) [![Hex version](https://img.shields.io/hexpm/v/ejabberd.svg "Hex version")](https://hex.pm/packages/ejabberd)
+
+ejabberd is a distributed, fault-tolerant technology that allows the creation
+of large-scale instant messaging applications. The server can reliably support
+thousands of simultaneous users on a single node and has been designed to
+provide exceptional standards of fault tolerance. As an open source
+technology, based on industry-standards, ejabberd can be used to build bespoke
+solutions very cost effectively.
+
+
+Key Features
+------------
+
+- **Cross-platform**  
+  ejabberd runs under Microsoft Windows and Unix-derived systems such as
+  Linux, FreeBSD and NetBSD.
+
+- **Distributed**  
+  You can run ejabberd on a cluster of machines and all of them will serve the
+  same XMPP domain(s). When you need more capacity you can simply add a new
+  cheap node to your cluster. Accordingly, you do not need to buy an expensive
+  high-end machine to support tens of thousands concurrent users.
+
+- **Fault-tolerant**  
+  You can deploy an ejabberd cluster so that all the information required for
+  a properly working service will be replicated permanently on all nodes. This
+  means that if one of the nodes crashes, the others will continue working
+  without disruption. In addition, nodes also can be added or replaced ‘on
+  the fly’.
+
+- **Administrator-friendly**  
+  ejabberd is built on top of the Open Source Erlang. As a result you do not
+  need to install an external database, an external web server, amongst others
+  because everything is already included, and ready to run out of the box.
+  Other administrator benefits include:
+  - Comprehensive documentation.
+  - Straightforward installers for Linux and Mac OS X.
+  - Web administration.
+  - Shared roster groups.
+  - Command line administration tool.
+  - Can integrate with existing authentication mechanisms.
+  - Capability to send announce messages.
+
+- **Internationalized**  
+  ejabberd leads in internationalization. Hence it is very well suited in a
+  globalized world. Related features are:
+  - Translated to 25 languages.
+  - Support for IDNA.
+
+- **Open Standards**  
+  ejabberd is the first Open Source Jabber server claiming to fully comply to
+  the XMPP standard.
+  - Fully XMPP-compliant.
+  - XML-based protocol.
+  - Many protocols supported.
+
+
+Additional Features
+-------------------
+
+Moreover, ejabberd comes with a wide range of other state-of-the-art features:
+
+- **Modularity**
+  - Load only the modules you want.
+  - Extend ejabberd with your own custom modules.
+
+- **Security**
+  - SASL and STARTTLS for c2s and s2s connections.
+  - STARTTLS and Dialback s2s connections.
+  - Web Admin accessible via HTTPS secure access.
+
+- **Databases**
+  - Internal database for fast deployment (Mnesia).
+  - Native MySQL support.
+  - Native PostgreSQL support.
+  - ODBC data storage support.
+  - Microsoft SQL Server support.
+
+- **Authentication**
+  - Internal authentication.
+  - PAM, LDAP and ODBC.
+  - External authentication script.
+
+- **Others**
+  - Support for virtual hosting.
+  - Compressing XML streams with Stream Compression (XEP-0138).
+  - Statistics via Statistics Gathering (XEP-0039).
+  - IPv6 support both for c2s and s2s connections.
+  - Multi-User Chat module with support for clustering and HTML logging.
+  - Users Directory based on users vCards.
+  - Publish-Subscribe component with support for Personal Eventing.
+  - Support for web clients: HTTP Polling and HTTP Binding (BOSH).
+  - Component support: interface with networks such as AIM, ICQ and MSN.
+
+
+Quickstart guide
+----------------
+
+### 0. Requirements
+
+To compile ejabberd you need:
+
+ - GNU Make.
+ - GCC.
+ - Libexpat ≥ 1.95.
+ - Libyaml ≥ 0.1.4.
+ - Erlang/OTP ≥ 19.0.
+ - OpenSSL ≥ 1.0.0.
+ - Zlib ≥ 1.2.3, for Stream Compression support (XEP-0138). Optional.
+ - PAM library. Optional. For Pluggable Authentication Modules (PAM).
+ - ImageMagick's Convert program. Optional. For CAPTCHA challenges.
+
+If your system splits packages in libraries and development headers, you must
+install the development packages also.
+
+### 1. Compile and install on *nix systems
+
+To compile ejabberd, execute the following commands.  The first one is only
+necessary if your source tree didn't come with a `configure` script (In this
+case you need autoconf installed).
+
+    ./autogen.sh
+    ./configure
+    make
+
+To install ejabberd, run this command with system administrator rights (root
+user):
+
+    sudo make install
+
+These commands will:
+
+- Install the configuration files in `/etc/ejabberd/`
+- Install ejabberd binary, header and runtime files in `/lib/ejabberd/`
+- Install the administration script: `/sbin/ejabberdctl`
+- Install ejabberd documentation in `/share/doc/ejabberd/`
+- Create a spool directory: `/var/lib/ejabberd/`
+- Create a directory for log files: `/var/log/ejabberd/`
+
+
+### 2. Start ejabberd
+
+You can use the `ejabberdctl` command line administration script to
+start and stop ejabberd. For example:
+
+    ejabberdctl start
+
+
+For detailed information please refer to the ejabberd Installation and
+Operation Guide available online and in the `doc` directory of the source
+tarball.
+
+
+Development
+-----------
+
+In order to assist in the development of ejabberd, and particularly the
+execution of the test suite, a Vagrant environment is available at
+https://github.com/processone/ejabberd-vagrant-dev.
+
+To start ejabberd in development mode from the repository directory, you can
+type a command like:
+
+    EJABBERD_CONFIG_PATH=ejabberd.yml erl -pa ebin -pa deps/*/ebin -pa test -pa deps/elixir/lib/*/ebin/ -s ejabberd
+
+Links
+-----
+
+- Documentation: https://docs.ejabberd.im
+- Community site: https://www.ejabberd.im
+- ejabberd commercial offering and support: https://www.process-one.net/en/ejabberd

config/ejabberd.exs

@@ -61,7 +61,6 @@ defmodule Ejabberd.ConfigFile do
     @opts [
       port: 5280,
       web_admin: true,
-      http_poll: true,
       http_bind: true,
       captcha: true]
   end

config/ejabberd.yml

@@ -1,666 +0,0 @@
-###
-###               ejabberd configuration file
-###
-###
-
-### The parameters used in this configuration file are explained in more detail
-### in the ejabberd Installation and Operation Guide.
-### Please consult the Guide in case of doubts, it is included with
-### your copy of ejabberd, and is also available online at
-### http://www.process-one.net/en/ejabberd/docs/
-
-### The configuration file is written in YAML.
-### Refer to http://en.wikipedia.org/wiki/YAML for the brief description.
-### However, ejabberd treats different literals as different types:
-###
-### - unquoted or single-quoted strings. They are called "atoms".
-###   Example: dog, 'Jupiter', '3.14159', YELLOW
-###
-### - numeric literals. Example: 3, -45.0, .0
-###
-### - quoted or folded strings.
-###   Examples of quoted string: "Lizzard", "orange".
-###   Example of folded string:
-###   > Art thou not Romeo,
-###     and a Montague?
-
-###   =======
-###   LOGGING
-
-##
-## loglevel: Verbosity of log files generated by ejabberd.
-## 0: No ejabberd log at all (not recommended)
-## 1: Critical
-## 2: Error
-## 3: Warning
-## 4: Info
-## 5: Debug
-##
-loglevel: 4
-
-##
-## rotation: Describe how to rotate logs. Either size and/or date can trigger
-## log rotation. Setting count to N keeps N rotated logs. Setting count to 0
-## does not disable rotation, it instead rotates the file and keeps no previous
-## versions around. Setting size to X rotate log when it reaches X bytes.
-## To disable rotation set the size to 0 and the date to ""
-## Date syntax is taken from the syntax newsyslog uses in newsyslog.conf.
-## Some examples:
-##  $D0     rotate every night at midnight
-##  $D23    rotate every day at 23:00 hr
-##  $W0D23  rotate every week on Sunday at 23:00 hr
-##  $W5D16  rotate every week on Friday at 16:00 hr
-##  $M1D0   rotate on the first day of every month at midnight
-##  $M5D6   rotate on every 5th day of the month at 6:00 hr
-##
-log_rotate_size: 10485760
-log_rotate_date: ""
-log_rotate_count: 1
-
-##
-## overload protection: If you want to limit the number of messages per second
-## allowed from error_logger, which is a good idea if you want to avoid a flood
-## of messages when system is overloaded, you can set a limit.
-## 100 is ejabberd's default.
-log_rate_limit: 100
-
-##
-## watchdog_admins: Only useful for developers: if an ejabberd process
-## consumes a lot of memory, send live notifications to these XMPP
-## accounts.
-##
-## watchdog_admins:
-##   - "bob@example.com"
-
-
-###   ================
-###   SERVED HOSTNAMES
-
-##
-## hosts: Domains served by ejabberd.
-## You can define one or several, for example:
-## hosts:
-##   - "example.net"
-##   - "example.com"
-##   - "example.org"
-##
-hosts:
-  - "localhost"
-
-##
-## route_subdomains: Delegate subdomains to other XMPP servers.
-## For example, if this ejabberd serves example.org and you want
-## to allow communication with an XMPP server called im.example.org.
-##
-## route_subdomains: s2s
-
-###   ===============
-###   LISTENING PORTS
-
-##
-## listen: The ports ejabberd will listen on, which service each is handled
-## by and what options to start it with.
-##
-listen:
-  -
-    port: 5222
-    module: ejabberd_c2s
-    ##
-    ## If TLS is compiled in and you installed a SSL
-    ## certificate, specify the full path to the
-    ## file and uncomment these lines:
-    ##
-    ## certfile: "/path/to/ssl.pem"
-    ## starttls: true
-    ##
-    ## To enforce TLS encryption for client connections,
-    ## use this instead of the "starttls" option:
-    ##
-    ## starttls_required: true
-    ##
-    ## Custom OpenSSL options
-    ##
-    ## protocol_options:
-    ##   - "no_sslv3"
-    ##   - "no_tlsv1"
-    max_stanza_size: 65536
-    shaper: c2s_shaper
-    access: c2s
-  -
-    port: 5269
-    module: ejabberd_s2s_in
-  ##
-  ## ejabberd_service: Interact with external components (transports, ...)
-  ##
-  ## -
-  ##   port: 8888
-  ##   module: ejabberd_service
-  ##   access: all
-  ##   shaper_rule: fast
-  ##   ip: "127.0.0.1"
-  ##   hosts:
-  ##     "icq.example.org":
-  ##       password: "secret"
-  ##     "sms.example.org":
-  ##       password: "secret"
-
-  ##
-  ## ejabberd_stun: Handles STUN Binding requests
-  ##
-  ## -
-  ##   port: 3478
-  ##   transport: udp
-  ##   module: ejabberd_stun
-
-  ##
-  ## To handle XML-RPC requests that provide admin credentials:
-  ##
-  ## -
-  ##   port: 4560
-  ##   module: ejabberd_xmlrpc
-  -
-    port: 5280
-    module: ejabberd_http
-    ## request_handlers:
-    ##   "/pub/archive": mod_http_fileserver
-    web_admin: true
-    http_poll: true
-    http_bind: true
-    ## register: true
-    captcha: true
-
-##
-## s2s_use_starttls: Enable STARTTLS + Dialback for S2S connections.
-## Allowed values are: false optional required required_trusted
-## You must specify a certificate file.
-##
-## s2s_use_starttls: optional
-
-##
-## s2s_certfile: Specify a certificate file.
-##
-## s2s_certfile: "/path/to/ssl.pem"
-
-## Custom OpenSSL options
-##
-## s2s_protocol_options:
-##   - "no_sslv3"
-##   - "no_tlsv1"
-
-##
-## domain_certfile: Specify a different certificate for each served hostname.
-##
-## host_config:
-##   "example.org":
-##     domain_certfile: "/path/to/example_org.pem"
-##   "example.com":
-##     domain_certfile: "/path/to/example_com.pem"
-
-##
-## S2S whitelist or blacklist
-##
-## Default s2s policy for undefined hosts.
-##
-## s2s_access: s2s
-
-##
-## Outgoing S2S options
-##
-## Preferred address families (which to try first) and connect timeout
-## in milliseconds.
-##
-## outgoing_s2s_families:
-##   - ipv4
-##   - ipv6
-## outgoing_s2s_timeout: 10000
-
-###   ==============
-###   AUTHENTICATION
-
-##
-## auth_method: Method used to authenticate the users.
-## The default method is the internal.
-## If you want to use a different method,
-## comment this line and enable the correct ones.
-##
-auth_method: internal
-
-##
-## Store the plain passwords or hashed for SCRAM:
-## auth_password_format: plain
-## auth_password_format: scram
-##
-## Define the FQDN if ejabberd doesn't detect it:
-## fqdn: "server3.example.com"
-
-##
-## Authentication using external script
-## Make sure the script is executable by ejabberd.
-##
-## auth_method: external
-## extauth_program: "/path/to/authentication/script"
-
-##
-## Authentication using ODBC
-## Remember to setup a database in the next section.
-##
-## auth_method: odbc
-
-##
-## Authentication using PAM
-##
-## auth_method: pam
-## pam_service: "pamservicename"
-
-##
-## Authentication using LDAP
-##
-## auth_method: ldap
-##
-## List of LDAP servers:
-## ldap_servers:
-##   - "localhost"
-##
-## Encryption of connection to LDAP servers:
-## ldap_encrypt: none
-## ldap_encrypt: tls
-##
-## Port to connect to on LDAP servers:
-## ldap_port: 389
-## ldap_port: 636
-##
-## LDAP manager:
-## ldap_rootdn: "dc=example,dc=com"
-##
-## Password of LDAP manager:
-## ldap_password: "******"
-##
-## Search base of LDAP directory:
-## ldap_base: "dc=example,dc=com"
-##
-## LDAP attribute that holds user ID:
-## ldap_uids:
-##   - "mail": "%u@mail.example.org"
-##
-## LDAP filter:
-## ldap_filter: "(objectClass=shadowAccount)"
-
-##
-## Anonymous login support:
-##   auth_method: anonymous
-##   anonymous_protocol: sasl_anon | login_anon | both
-##   allow_multiple_connections: true | false
-##
-## host_config:
-##   "public.example.org":
-##     auth_method: anonymous
-##     allow_multiple_connections: false
-##     anonymous_protocol: sasl_anon
-##
-## To use both anonymous and internal authentication:
-##
-## host_config:
-##   "public.example.org":
-##     auth_method:
-##       - internal
-##       - anonymous
-
-###   ==============
-###   DATABASE SETUP
-
-## ejabberd by default uses the internal Mnesia database,
-## so you do not necessarily need this section.
-## This section provides configuration examples in case
-## you want to use other database backends.
-## Please consult the ejabberd Guide for details on database creation.
-
-##
-## MySQL server:
-##
-## odbc_type: mysql
-## odbc_server: "server"
-## odbc_database: "database"
-## odbc_username: "username"
-## odbc_password: "password"
-##
-## If you want to specify the port:
-## odbc_port: 1234
-
-##
-## PostgreSQL server:
-##
-## odbc_type: pgsql
-## odbc_server: "server"
-## odbc_database: "database"
-## odbc_username: "username"
-## odbc_password: "password"
-##
-## If you want to specify the port:
-## odbc_port: 1234
-##
-## If you use PostgreSQL, have a large database, and need a
-## faster but inexact replacement for "select count(*) from users"
-##
-## pgsql_users_number_estimate: true
-
-##
-## ODBC compatible or MSSQL server:
-##
-## odbc_type: odbc
-## odbc_server: "DSN=ejabberd;UID=ejabberd;PWD=ejabberd"
-
-##
-## Number of connections to open to the database for each virtual host
-##
-## odbc_pool_size: 10
-
-##
-## Interval to make a dummy SQL request to keep the connections to the
-## database alive. Specify in seconds: for example 28800 means 8 hours
-##
-## odbc_keepalive_interval: undefined
-
-###   ===============
-###   TRAFFIC SHAPERS
-
-shaper:
-  ##
-  ## The "normal" shaper limits traffic speed to 1000 B/s
-  ##
-  normal: 1000
-
-  ##
-  ## The "fast" shaper limits traffic speed to 50000 B/s
-  ##
-  fast: 50000
-
-##
-## This option specifies the maximum number of elements in the queue
-## of the FSM. Refer to the documentation for details.
-##
-max_fsm_queue: 1000
-
-###.   ====================
-###'   ACCESS CONTROL LISTS
-acl:
-  ##
-  ## The 'admin' ACL grants administrative privileges to XMPP accounts.
-  ## You can put here as many accounts as you want.
-  ##
-  ## admin:
-  ##   user:
-  ##     - "aleksey": "localhost"
-  ##     - "ermine": "example.org"
-  ##
-  ## Blocked users
-  ##
-  ## blocked:
-  ##   user:
-  ##     - "baduser": "example.org"
-  ##     - "test"
-
-  ## Local users: don't modify this.
-  ##
-  local:
-    user_regexp: ""
-
-  ##
-  ## More examples of ACLs
-  ##
-  ## jabberorg:
-  ##   server:
-  ##     - "jabber.org"
-  ## aleksey:
-  ##   user:
-  ##     - "aleksey": "jabber.ru"
-  ## test:
-  ##   user_regexp: "^test"
-  ##   user_glob: "test*"
-
-  ##
-  ## Loopback network
-  ##
-  loopback:
-    ip:
-      - "127.0.0.0/8"
-
-  ##
-  ## Bad XMPP servers
-  ##
-  ## bad_servers:
-  ##   server:
-  ##     - "xmpp.zombie.org"
-  ##     - "xmpp.spam.com"
-
-##
-## Define specific ACLs in a virtual host.
-##
-## host_config:
-##   "localhost":
-##     acl:
-##       admin:
-##         user:
-##           - "bob-local": "localhost"
-
-###   ============
-###   ACCESS RULES
-access:
-  ## Maximum number of simultaneous sessions allowed for a single user:
-  max_user_sessions:
-    all: 10
-  ## Maximum number of offline messages that users can have:
-  max_user_offline_messages:
-    admin: 5000
-    all: 100
-  ## This rule allows access only for local users:
-  local:
-    local: allow
-  ## Only non-blocked users can use c2s connections:
-  c2s:
-    blocked: deny
-    all: allow
-  ## For C2S connections, all users except admins use the "normal" shaper
-  c2s_shaper:
-    admin: none
-    all: normal
-  ## All S2S connections use the "fast" shaper
-  s2s_shaper:
-    all: fast
-  ## Only admins can send announcement messages:
-  announce:
-    admin: allow
-  ## Only admins can use the configuration interface:
-  configure:
-    admin: allow
-  ## Admins of this server are also admins of the MUC service:
-  muc_admin:
-    admin: allow
-  ## Only accounts of the local ejabberd server can create rooms:
-  muc_create:
-    local: allow
-  ## All users are allowed to use the MUC service:
-  muc:
-    all: allow
-  ## Only accounts on the local ejabberd server can create Pubsub nodes:
-  pubsub_createnode:
-    local: allow
-  ## In-band registration allows registration of any possible username.
-  ## To disable in-band registration, replace 'allow' with 'deny'.
-  register:
-    all: allow
-  ## Only allow to register from localhost
-  trusted_network:
-    loopback: allow
-  ## Do not establish S2S connections with bad servers
-  ## s2s:
-  ##   bad_servers: deny
-  ##   all: allow
-
-## By default the frequency of account registrations from the same IP
-## is limited to 1 account every 10 minutes. To disable, specify: infinity
-## registration_timeout: 600
-
-##
-## Define specific Access Rules in a virtual host.
-##
-## host_config:
-##   "localhost":
-##     access:
-##       c2s:
-##         admin: allow
-##         all: deny
-##       register:
-##         all: deny
-
-###   ================
-###   DEFAULT LANGUAGE
-
-##
-## language: Default language used for server messages.
-##
-language: "en"
-
-##
-## Set a different default language in a virtual host.
-##
-## host_config:
-##   "localhost":
-##     language: "ru"
-
-###   =======
-###   CAPTCHA
-
-##
-## Full path to a script that generates the image.
-##
-## captcha_cmd: "/lib/ejabberd/priv/bin/captcha.sh"
-
-##
-## Host for the URL and port where ejabberd listens for CAPTCHA requests.
-##
-## captcha_host: "example.org:5280"
-
-##
-## Limit CAPTCHA calls per minute for JID/IP to avoid DoS.
-##
-## captcha_limit: 5
-
-###   =======
-###   MODULES
-
-##
-## Modules enabled in all ejabberd virtual hosts.
-##
-modules:
-  mod_adhoc: {}
-  ## mod_admin_extra: {}
-  mod_announce: # recommends mod_adhoc
-    access: announce
-  mod_blocking: {} # requires mod_privacy
-  mod_caps: {}
-  mod_carboncopy: {}
-  mod_client_state:
-    queue_chat_states: true
-    queue_presence: false
-  mod_configure: {} # requires mod_adhoc
-  mod_disco: {}
-  ## mod_echo: {}
-  mod_http_bind: {}
-  ## mod_http_fileserver:
-  ##   docroot: "/var/www"
-  ##   accesslog: "/var/log/ejabberd/access.log"
-  mod_last: {}
-  mod_muc:
-    ## host: "conference.@HOST@"
-    access: muc
-    access_create: muc_create
-    access_persistent: muc_create
-    access_admin: muc_admin
-  ## mod_muc_log: {}
-  mod_offline:
-    access_max_user_messages: max_user_offline_messages
-  mod_ping: {}
-  ## mod_pres_counter:
-  ##   count: 5
-  ##   interval: 60
-  mod_privacy: {}
-  mod_private: {}
-  ## mod_proxy65: {}
-  mod_pubsub:
-    access_createnode: pubsub_createnode
-    ## reduces resource comsumption, but XEP incompliant
-    ignore_pep_from_offline: true
-    ## XEP compliant, but increases resource comsumption
-    ## ignore_pep_from_offline: false
-    last_item_cache: false
-    plugins:
-      - "flat"
-      - "hometree"
-      - "pep" # pep requires mod_caps
-  mod_register:
-    ##
-    ## Protect In-Band account registrations with CAPTCHA.
-    ##
-    ## captcha_protected: true
-
-    ##
-    ## Set the minimum informational entropy for passwords.
-    ##
-    ## password_strength: 32
-
-    ##
-    ## After successful registration, the user receives
-    ## a message with this subject and body.
-    ##
-    welcome_message:
-      subject: "Welcome!"
-      body: |-
-        Hi.
-        Welcome to this XMPP server.
-
-    ##
-    ## When a user registers, send a notification to
-    ## these XMPP accounts.
-    ##
-    ## registration_watchers:
-    ##   - "admin1@example.org"
-
-    ##
-    ## Only clients in the server machine can register accounts
-    ##
-    ip_access: trusted_network
-
-    ##
-    ## Local c2s or remote s2s users cannot register accounts
-    ##
-    ## access_from: deny
-
-    access: register
-  mod_roster: {}
-  mod_shared_roster: {}
-  mod_stats: {}
-  mod_time: {}
-  mod_vcard: {}
-  mod_version: {}
-
-##
-## Enable modules with custom options in a specific virtual host
-##
-## host_config:
-##   "localhost":
-##     modules:
-##       mod_echo:
-##         host: "mirror.localhost"
-
-##
-## Enable modules management via ejabberdctl for installation and
-## uninstallation of public/private contributed modules
-## (enabled by default)
-##
-
-allow_contrib_modules: true
-
-### Local Variables:
-### mode: yaml
-### End:
-### vim: set filetype=yaml tabstop=8

ejabberd.yml.example

@@ -36,7 +36,8 @@ log_rotate_count: 1
 log_rate_limit: 100
 
 certfiles:
-  - "/etc/letsencrypt/live/*/*.pem"
+  - "/etc/letsencrypt/live/localhost/fullchain.pem"
+  - "/etc/letsencrypt/live/localhost/privkey.pem"
 
 listen:
   -
@@ -178,13 +179,16 @@ modules:
   mod_ping: {}
   mod_privacy: {}
   mod_private: {}
+  mod_proxy65:
+    access: local
+    max_connections: 5
   mod_pubsub:
     access_createnode: pubsub_createnode
     plugins:
       - "flat"
       - "pep"
     force_node_config:
-      ## Comment out the following lines to enable OMEMO support
+      ## Change from "whitelist" to "open" to enable OMEMO support
       ## See https://github.com/processone/ejabberd/issues/2425
       "eu.siacs.conversations.axolotl.*":
         access_model: whitelist

include/mod_carboncopy.hrl

@@ -1,27 +0,0 @@
-%%%----------------------------------------------------------------------
-%%%
-%%% ejabberd, Copyright (C) 2002-2018   ProcessOne
-%%%
-%%% This program is free software; you can redistribute it and/or
-%%% modify it under the terms of the GNU General Public License as
-%%% published by the Free Software Foundation; either version 2 of the
-%%% License, or (at your option) any later version.
-%%%
-%%% This program is distributed in the hope that it will be useful,
-%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-%%% General Public License for more details.
-%%%
-%%% You should have received a copy of the GNU General Public License along
-%%% with this program; if not, write to the Free Software Foundation, Inc.,
-%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-%%%
-%%%----------------------------------------------------------------------
-
--type matchspec_atom() :: '_' | '$1' | '$2' | '$3' | '$4'.
--record(carboncopy, {us       :: {binary(), binary()} | matchspec_atom(), 
-		     resource :: binary() | matchspec_atom(),
-		     version  :: binary() | matchspec_atom(),
-		     node = node() :: node() | matchspec_atom()}).
-
--define(CARBONCOPY_CACHE, carboncopy_cache).

include/mod_muc_room.hrl

@@ -22,8 +22,6 @@
 
 -define(SETS, gb_sets).
 
--define(DICT, dict).
-
 -record(lqueue,
 {
     queue   :: p1_queue:queue(),
@@ -105,13 +103,13 @@
     access                  = {none,none,none,none} :: {atom(), atom(), atom(), atom()},
     jid                     = #jid{} :: jid(),
     config                  = #config{} :: config(),
-    users                   = (?DICT):new() :: dict:dict(),
-    subscribers             = (?DICT):new() :: dict:dict(),
-    subscriber_nicks        = (?DICT):new() :: dict:dict(),
+    users                   = #{} :: map(),
+    subscribers             = #{} :: map(),
+    subscriber_nicks        = #{} :: map(),
     last_voice_request_time = treap:empty() :: treap:treap(),
-    robots                  = (?DICT):new() :: dict:dict(),
-    nicks                   = (?DICT):new() :: dict:dict(),
-    affiliations            = (?DICT):new() :: dict:dict(),
+    robots                  = #{} :: map(),
+    nicks                   = #{} :: map(),
+    affiliations            = #{} :: map(),
     history                 :: lqueue(),
     subject                 = [] :: [text()],
     subject_author          = <<"">> :: binary(),

mix.exs

@@ -3,7 +3,7 @@ defmodule Ejabberd.Mixfile do
 
   def project do
     [app: :ejabberd,
-     version: "18.6.0",
+     version: "18.12.0",
      description: description(),
      elixir: "~> 1.4",
      elixirc_paths: ["lib"],
@@ -29,7 +29,7 @@ defmodule Ejabberd.Mixfile do
      included_applications: [:lager, :mnesia, :inets, :p1_utils, :cache_tab,
                              :fast_tls, :stringprep, :fast_xml, :xmpp,
                              :stun, :fast_yaml, :esip, :jiffy, :p1_oauth2,
-                             :eimp, :base64url, :jose]
+                             :eimp, :base64url, :jose, :pkix]
                          ++ cond_apps()]
   end
 
@@ -58,7 +58,7 @@ defmodule Ejabberd.Mixfile do
   end
 
   defp deps do
-    [{:lager, "~> 3.4.0"},
+    [{:lager, "~> 3.6.0"},
      {:p1_utils, "~> 1.0"},
      {:fast_xml, "~> 1.1"},
      {:xmpp, "~> 1.2"},
@@ -73,6 +73,7 @@ defmodule Ejabberd.Mixfile do
      {:jiffy, "~> 0.14.7"},
      {:p1_oauth2, "~> 0.6.1"},
      {:distillery, "~> 1.0"},
+     {:pkix, "~> 1.0"},
      {:ex_doc, ">= 0.0.0", only: :dev},
      {:eimp, "~> 1.0"},
      {:base64url, "~> 0.0.1"},

mix.lock

@@ -1,15 +1,15 @@
 %{
   "base64url": {:hex, :base64url, "0.0.1", "36a90125f5948e3afd7be97662a1504b934dd5dac78451ca6e9abf85a10286be", [:rebar], [], "hexpm"},
-  "cache_tab": {:hex, :cache_tab, "1.0.16", "0223820e5071d3252b9921db9dcc74a09ec8a660120271fdb47c3c40b6b52bee", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
-  "distillery": {:hex, :distillery, "1.5.3", "b2f4fc34ec71ab4f1202a796f9290e068883b042319aa8c9aa45377ecac8597a", [:mix], [], "hexpm"},
-  "earmark": {:hex, :earmark, "1.2.5", "4d21980d5d2862a2e13ec3c49ad9ad783ffc7ca5769cf6ff891a4553fbaae761", [:mix], [], "hexpm"},
+  "cache_tab": {:hex, :cache_tab, "1.0.17", "0020e1036d7074d83a71be28b329ceb3e7f9d41cc5a8529b06c32ce4d8ee4995", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
+  "distillery": {:hex, :distillery, "1.5.5", "c6132d5c0152bdce6850fb6c942d58f1971b169b6965d908dc4e8767cfa51a95", [:mix], [], "hexpm"},
+  "earmark": {:hex, :earmark, "1.3.0", "17f0c38eaafb4800f746b457313af4b2442a8c2405b49c645768680f900be603", [:mix], [], "hexpm"},
   "eimp": {:hex, :eimp, "1.0.9", "daf0d2904be3ef5e4128d946e158113cdb4d52555023746d29b83ce86b531f3c", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
   "epam": {:hex, :epam, "1.0.4", "2a5e40cbf9b2cf41df515782894c3b33c81b8ad32fff2fc847c3f725071dfaed", [:rebar3], [], "hexpm"},
   "eredis": {:hex, :eredis, "1.1.0", "8d8d74496f35216679b97726b75fb1c8715e99dd7f3ef9f9824a2264c3e0aac0", [:rebar3], [], "hexpm"},
-  "esip": {:hex, :esip, "1.0.26", "b50c92f8ac3e8e8ba901f0a6cc7e0e47fdc832b0f3044eddb6032ca26845cf97", [:rebar3], [{:fast_tls, "1.0.25", [hex: :fast_tls, repo: "hexpm", optional: false]}, {:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}, {:stun, "1.0.25", [hex: :stun, repo: "hexpm", optional: false]}], "hexpm"},
-  "ex_doc": {:hex, :ex_doc, "0.18.3", "f4b0e4a2ec6f333dccf761838a4b253d75e11f714b85ae271c9ae361367897b7", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}], "hexpm"},
+  "esip": {:hex, :esip, "1.0.27", "13a94542b659a9b3e4e013aedaf2f6a92de53d35945902d693657a67c6955b83", [:rebar3], [{:fast_tls, "1.0.26", [hex: :fast_tls, repo: "hexpm", optional: false]}, {:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}, {:stun, "1.0.26", [hex: :stun, repo: "hexpm", optional: false]}], "hexpm"},
+  "ex_doc": {:hex, :ex_doc, "0.19.1", "519bb9c19526ca51d326c060cb1778d4a9056b190086a8c6c115828eaccea6cf", [:mix], [{:earmark, "~> 1.1", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.7", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm"},
   "ezlib": {:hex, :ezlib, "1.0.4", "2434e4bb549cb060d5ac02261ba48fbe1a69b2ae4e1bf7485a3b27b3f3ec618d", [:rebar3], [], "hexpm"},
-  "fast_tls": {:hex, :fast_tls, "1.0.25", "cbf875fe709d6fd03d3266c920bfe15f4d22736535d73421300cebf9d86bd851", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
+  "fast_tls": {:hex, :fast_tls, "1.0.26", "38d78859ca56e8600aca3ef73137582a279a280d71f7581c64a1eddbde38accb", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
   "fast_xml": {:hex, :fast_xml, "1.1.34", "d76fc639d3607a44c4f0fb2dfdee1067b6c37b02b39706d8f067cb77eb2f6016", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
   "fast_yaml": {:hex, :fast_yaml, "1.0.17", "e945ef64e0cb7c311c7b42804dbe32a24e13a2afc0ffe249b7e0f9f9ac08e176", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
   "goldrush": {:hex, :goldrush, "0.1.9", "f06e5d5f1277da5c413e84d5a2924174182fb108dabb39d5ec548b27424cd106", [:rebar3], [], "hexpm"},
@@ -17,19 +17,23 @@
   "iconv": {:hex, :iconv, "1.0.10", "fc7de75c0a1fbc1e4ed0c68637ae7464a5dd9eee81710fff26321922d144ecea", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
   "jiffy": {:hex, :jiffy, "0.14.13", "225a9a35e26417832c611526567194b4d3adc4f0dfa5f2f7008f4684076f2a01", [:rebar3], [], "hexpm"},
   "jose": {:hex, :jose, "1.8.4", "7946d1e5c03a76ac9ef42a6e6a20001d35987afd68c2107bcd8f01a84e75aa73", [:mix, :rebar3], [{:base64url, "~> 0.0.1", [hex: :base64url, repo: "hexpm", optional: false]}], "hexpm"},
-  "lager": {:hex, :lager, "3.4.2", "150b9a17b23ae6d3265cc10dc360747621cf217b7a22b8cddf03b2909dbf7aa5", [:rebar3], [{:goldrush, "0.1.9", [hex: :goldrush, repo: "hexpm", optional: false]}], "hexpm"},
+  "lager": {:hex, :lager, "3.6.7", "2fbf823944caa0fc10df5ec13f3f047524a249bb32f0d801b7900c9610264286", [:rebar3], [{:goldrush, "0.1.9", [hex: :goldrush, repo: "hexpm", optional: false]}], "hexpm"},
   "luerl": {:hex, :luerl, "0.3.1", "5412807630aac1aaf59ffe5a1bc09259c447b4faeb1d3fe2d4ef41b87676cb04", [:rebar3], [], "hexpm"},
-  "meck": {:hex, :meck, "0.8.10", "455aaef8403be46752272206613e7a15467c014d40994b22fb54cde4d1ff7075", [:rebar3], [], "hexpm"},
+  "makeup": {:hex, :makeup, "0.5.5", "9e08dfc45280c5684d771ad58159f718a7b5788596099bdfb0284597d368a882", [:mix], [{:nimble_parsec, "~> 0.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm"},
+  "makeup_elixir": {:hex, :makeup_elixir, "0.10.0", "0f09c2ddf352887a956d84f8f7e702111122ca32fbbc84c2f0569b8b65cbf7fa", [:mix], [{:makeup, "~> 0.5.5", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm"},
+  "meck": {:hex, :meck, "0.8.12", "1f7b1a9f5d12c511848fec26bbefd09a21e1432eadb8982d9a8aceb9891a3cf2", [:rebar3], [], "hexpm"},
   "moka": {:git, "https://github.com/processone/moka.git", "3eed3a6dd7dedb70a6cd18f86c7561a18626eb3b", [tag: "1.0.5c"]},
-  "p1_mysql": {:hex, :p1_mysql, "1.0.7", "9fbadf8fa283ae8657faa4f6bbe13f2e3b9da0cdcfbc699cfc120d0905282056", [:rebar3], [], "hexpm"},
+  "nimble_parsec": {:hex, :nimble_parsec, "0.4.0", "ee261bb53214943679422be70f1658fff573c5d0b0a1ecd0f18738944f818efe", [:mix], [], "hexpm"},
+  "p1_mysql": {:hex, :p1_mysql, "1.0.8", "34ed5fe2f0e16a6ee5805c0c6c1d30ffbc4c5c9753197cdf384ee6e82c57b506", [:rebar3], [], "hexpm"},
   "p1_oauth2": {:hex, :p1_oauth2, "0.6.3", "fbd91ba86bd7f03d2a4f6e62affa86bab9930abfd6b473d61eefb148f246cd46", [:rebar3], [], "hexpm"},
   "p1_pgsql": {:hex, :p1_pgsql, "1.1.6", "631004602b06ca6f55d759001f180185685c7097e583f3b0f7dd9b8e05ba5db1", [:rebar3], [], "hexpm"},
   "p1_utils": {:hex, :p1_utils, "1.0.13", "176577cafb54a8c2fdc0a7fc62b9a21ab0f66588f4062792cd9e65f3e500bfdb", [:rebar3], [], "hexpm"},
+  "pkix": {:hex, :pkix, "1.0.0", "d88658eccc30227e929efa91c6ca6a4d2b4d40b4db3635ebd6ed9e246ecfcf82", [:rebar3], [], "hexpm"},
   "riak_pb": {:hex, :riak_pb, "2.3.2", "48ffbf66dbb3f136ab9a7134bac4e496754baa5ef58c4f50a61326736d996390", [:make, :mix, :rebar3], [{:hamcrest, "~> 0.4.1", [hex: :basho_hamcrest, repo: "hexpm", optional: false]}], "hexpm"},
   "riakc": {:hex, :riakc, "2.5.3", "6132d9e687a0dfd314b2b24c4594302ca8b55568a5d733c491d8fb6cd4004763", [:make, :mix, :rebar3], [{:riak_pb, "~> 2.3", [hex: :riak_pb, repo: "hexpm", optional: false]}], "hexpm"},
   "samerlib": {:git, "https://github.com/processone/samerlib", "fbbba035b1548ac4e681df00d61bf609645333a0", [tag: "0.8.0c"]},
   "sqlite3": {:hex, :sqlite3, "1.1.6", "4ea71af0b45908b5f02c9b09e4c87177039ef404f20accb35049cd8924cc417c", [:rebar3], [], "hexpm"},
   "stringprep": {:hex, :stringprep, "1.0.14", "230a2d1c576bba168749d653fd6681166d02431ef07161a918444f3edb478ad0", [:rebar3], [{:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
-  "stun": {:hex, :stun, "1.0.25", "e324c94c28d636578db79eb26979cd07140f0dabcdc0d5b197650ba0bc44a31a", [:rebar3], [{:fast_tls, "1.0.25", [hex: :fast_tls, repo: "hexpm", optional: false]}, {:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
-  "xmpp": {:hex, :xmpp, "1.2.5", "98ae86706013e51349e962b67c30293d14672603b5c7d782b2c79b52ceaa659f", [:rebar3], [{:ezlib, "1.0.4", [hex: :ezlib, repo: "hexpm", optional: false]}, {:fast_tls, "1.0.25", [hex: :fast_tls, repo: "hexpm", optional: false]}, {:fast_xml, "1.1.34", [hex: :fast_xml, repo: "hexpm", optional: false]}, {:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}, {:stringprep, "1.0.14", [hex: :stringprep, repo: "hexpm", optional: false]}], "hexpm"},
+  "stun": {:hex, :stun, "1.0.26", "87b05229d0519f0db5c6b67b5c25ed3b79766beb96eba83d29bde4cae9e702e4", [:rebar3], [{:fast_tls, "1.0.26", [hex: :fast_tls, repo: "hexpm", optional: false]}, {:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}], "hexpm"},
+  "xmpp": {:hex, :xmpp, "1.2.7", "79bbe73f4172c8a4ea3cf8ae81c85f395cc651aed446b4d2646f7e9f55e12d74", [:rebar3], [{:ezlib, "1.0.4", [hex: :ezlib, repo: "hexpm", optional: false]}, {:fast_tls, "1.0.26", [hex: :fast_tls, repo: "hexpm", optional: false]}, {:fast_xml, "1.1.34", [hex: :fast_xml, repo: "hexpm", optional: false]}, {:p1_utils, "1.0.13", [hex: :p1_utils, repo: "hexpm", optional: false]}, {:stringprep, "1.0.14", [hex: :stringprep, repo: "hexpm", optional: false]}], "hexpm"},
 }

rebar.config

@@ -18,23 +18,23 @@
 %%%
 %%%----------------------------------------------------------------------
 
-{deps, [{lager, ".*", {git, "https://github.com/erlang-lager/lager",
-                       {tag, {if_version_above, "17", "3.6.5", "3.2.1"}}}},
+{deps, [{lager, ".*", {git, "https://github.com/erlang-lager/lager", "3.6.7"}},
         {p1_utils, ".*", {git, "https://github.com/processone/p1_utils", {tag, "1.0.13"}}},
-        {cache_tab, ".*", {git, "https://github.com/processone/cache_tab", {tag, "1.0.16"}}},
-        {fast_tls, ".*", {git, "https://github.com/processone/fast_tls", {tag, "1.0.25"}}},
+        {cache_tab, ".*", {git, "https://github.com/processone/cache_tab", {tag, "1.0.17"}}},
+        {fast_tls, ".*", {git, "https://github.com/processone/fast_tls", {tag, "1.0.26"}}},
         {stringprep, ".*", {git, "https://github.com/processone/stringprep", {tag, "1.0.14"}}},
         {fast_xml, ".*", {git, "https://github.com/processone/fast_xml", {tag, "1.1.34"}}},
-        {xmpp, ".*", {git, "https://github.com/processone/xmpp", {tag, "1.2.5"}}},
+        {xmpp, ".*", {git, "https://github.com/processone/xmpp", {tag, "1.2.7"}}},
         {fast_yaml, ".*", {git, "https://github.com/processone/fast_yaml", {tag, "1.0.17"}}},
         {jiffy, ".*", {git, "https://github.com/davisp/jiffy", {tag, "0.14.8"}}},
         {p1_oauth2, ".*", {git, "https://github.com/processone/p1_oauth2", {tag, "0.6.3"}}},
+	{pkix, ".*", {git, "https://github.com/processone/pkix", {tag, "1.0.0"}}},
         {jose, ".*", {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.8.4"}}},
-        {eimp, ".*", {git, "https://github.com/processone/eimp", {tag, "1.0.8"}}},
-        {if_var_true, stun, {stun, ".*", {git, "https://github.com/processone/stun", {tag, "1.0.25"}}}},
-        {if_var_true, sip, {esip, ".*", {git, "https://github.com/processone/esip", {tag, "1.0.26"}}}},
+        {eimp, ".*", {git, "https://github.com/processone/eimp", {tag, "1.0.9"}}},
+        {if_var_true, stun, {stun, ".*", {git, "https://github.com/processone/stun", {tag, "1.0.26"}}}},
+        {if_var_true, sip, {esip, ".*", {git, "https://github.com/processone/esip", {tag, "1.0.27"}}}},
         {if_var_true, mysql, {p1_mysql, ".*", {git, "https://github.com/processone/p1_mysql",
-                                               {tag, "1.0.7"}}}},
+                                               {tag, "1.0.8"}}}},
         {if_var_true, pgsql, {p1_pgsql, ".*", {git, "https://github.com/processone/p1_pgsql",
                                                {tag, "1.1.6"}}}},
         {if_var_true, sqlite, {sqlite3, ".*", {git, "https://github.com/processone/erlang-sqlite3",
@@ -56,7 +56,7 @@
         {if_var_true, tools, {luerl, ".*", {git, "https://github.com/rvirding/luerl",
                                             {tag, "v0.3"}}}},
         {if_var_true, tools, {meck, "0.8.*", {git, "https://github.com/eproxus/meck",
-                                              {tag, "0.8.4"}}}},
+                                              {tag, "0.8.12"}}}},
         {if_var_true, redis, {eredis, ".*", {git, "https://github.com/wooga/eredis",
                                              {tag, "v1.0.8"}}}}]}.
 
@@ -76,6 +76,7 @@
                   epam,
                   ezlib,
                   eimp,
+                  pkix,
                   iconv]}}.
 
 {erl_first_files, ["src/ejabberd_sql_pt.erl", "src/ejabberd_config.erl",
@@ -94,10 +95,6 @@
             {if_var_true, roster_gateway_workaround, {d, 'ROSTER_GATWAY_WORKAROUND'}},
             {if_var_match, db_type, mssql, {d, 'mssql'}},
             {if_var_true, elixir, {d, 'ELIXIR_ENABLED'}},
-            {if_have_fun, {crypto, strong_rand_bytes, 1}, {d, 'STRONG_RAND_BYTES'}},
-            {if_have_fun, {rand, uniform, 1}, {d, 'RAND_UNIFORM'}},
-            {if_have_fun, {gb_sets, iterator_from, 2}, {d, 'GB_SETS_ITERATOR_FROM'}},
-            {if_have_fun, {public_key, short_name_hash, 1}, {d, 'SHORT_NAME_HASH'}},
             {if_var_true, new_sql_schema, {d, 'NEW_SQL_SCHEMA'}},
             {if_var_true, hipe, native},
             {src_dirs, [src,

sql/lite.new.sql

@@ -387,17 +387,6 @@ CREATE TABLE bosh (
 
 CREATE UNIQUE INDEX i_bosh_sid ON bosh(sid);
 
-CREATE TABLE carboncopy (
-    username text NOT NULL,
-    server_host text NOT NULL,
-    resource text NOT NULL,
-    namespace text NOT NULL,
-    node text NOT NULL,
-    PRIMARY KEY (server_host, username, resource)
-);
-
-CREATE INDEX i_carboncopy_sh_user ON carboncopy (server_host, username);
-
 CREATE TABLE proxy65 (
     sid text NOT NULL,
     pid_t text NOT NULL,

sql/lite.sql

@@ -357,16 +357,6 @@ CREATE TABLE bosh (
 
 CREATE UNIQUE INDEX i_bosh_sid ON bosh(sid);
 
-CREATE TABLE carboncopy (
-    username text NOT NULL,
-    resource text NOT NULL,
-    namespace text NOT NULL,
-    node text NOT NULL
-);
-
-CREATE UNIQUE INDEX i_carboncopy_ur ON carboncopy (username, resource);
-CREATE INDEX i_carboncopy_user ON carboncopy (username);
-
 CREATE TABLE proxy65 (
     sid text NOT NULL,
     pid_t text NOT NULL,

sql/mssql.sql

@@ -531,19 +531,6 @@ CREATE TABLE [dbo].[bosh] (
 )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON)
 );
 
-CREATE TABLE [dbo].[carboncopy] (
-    [username] [varchar] (255) NOT NULL,
-    [resource] [varchar] (255) NOT NULL,
-    [namespace] [varchar] (255) NOT NULL,
-    [node] [varchar] (255) NOT NULL
-);
-
-CREATE UNIQUE CLUSTERED INDEX [carboncopy_ur] ON [carboncopy] (username, resource)
-WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
-
-CREATE INDEX [carboncopy_user] ON [carboncopy] (username)
-WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
-
 CREATE TABLE [dbo].[push_session] (
     [username] [varchar] (255) NOT NULL,
     [timestamp] [bigint] NOT NULL,

sql/mysql.new.sql

@@ -276,7 +276,7 @@ CREATE TABLE pubsub_item (
   publisher text NOT NULL,
   creation varchar(32) NOT NULL,
   modification varchar(32) NOT NULL,
-  payload text NOT NULL
+  payload mediumtext NOT NULL
 ) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 CREATE INDEX i_pubsub_item_itemid ON pubsub_item(itemid(36));
 CREATE UNIQUE INDEX i_pubsub_item_tuple ON pubsub_item(nodeid, itemid(36));
@@ -403,17 +403,6 @@ CREATE TABLE bosh (
 
 CREATE UNIQUE INDEX i_bosh_sid ON bosh(sid(75));
 
-CREATE TABLE carboncopy (
-    username text NOT NULL,
-    server_host text NOT NULL,
-    resource text NOT NULL,
-    namespace text NOT NULL,
-    node text NOT NULL,
-    PRIMARY KEY (server_host(191), username(191), resource(191))
-) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-
-CREATE INDEX i_carboncopy_sh_user ON carboncopy (server_host(191), username(75));
-
 CREATE TABLE proxy65 (
     sid text NOT NULL,
     pid_t text NOT NULL,

sql/mysql.sql

@@ -253,7 +253,7 @@ CREATE TABLE pubsub_item (
   publisher text NOT NULL,
   creation varchar(32) NOT NULL,
   modification varchar(32) NOT NULL,
-  payload text NOT NULL
+  payload mediumtext NOT NULL
 ) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
 CREATE INDEX i_pubsub_item_itemid ON pubsub_item(itemid(36));
 CREATE UNIQUE INDEX i_pubsub_item_tuple ON pubsub_item(nodeid, itemid(36));
@@ -373,16 +373,6 @@ CREATE TABLE bosh (
 
 CREATE UNIQUE INDEX i_bosh_sid ON bosh(sid(75));
 
-CREATE TABLE carboncopy (
-    username text NOT NULL,
-    resource text NOT NULL,
-    namespace text NOT NULL,
-    node text NOT NULL
-) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
-
-CREATE UNIQUE INDEX i_carboncopy_ur ON carboncopy (username(75), resource(75));
-CREATE INDEX i_carboncopy_user ON carboncopy (username(75));
-
 CREATE TABLE proxy65 (
     sid text NOT NULL,
     pid_t text NOT NULL,

sql/pg.new.sql

@@ -156,13 +156,6 @@
 -- CREATE INDEX i_sm_sh_username ON sm USING btree (server_host, username);
 -- ALTER TABLE sm ALTER COLUMN server_host DROP DEFAULT;
 
--- ALTER TABLE carboncopy ADD COLUMN server_host text NOT NULL DEFAULT '<HOST>';
--- DROP INDEX i_carboncopy_ur;
--- DROP INDEX i_carboncopy_user;
--- ALTER TABLE carboncopy ADD PRIMARY KEY (server_host, username, resource);
--- CREATE INDEX i_carboncopy_sh_user ON carboncopy USING btree (server_host, username);
--- ALTER TABLE carboncopy ALTER COLUMN server_host DROP DEFAULT;
-
 
 CREATE TABLE users (
     username text NOT NULL,
@@ -555,17 +548,6 @@ CREATE TABLE bosh (
 
 CREATE UNIQUE INDEX i_bosh_sid ON bosh USING btree (sid);
 
-CREATE TABLE carboncopy (
-    username text NOT NULL,
-    server_host text NOT NULL,
-    resource text NOT NULL,
-    namespace text NOT NULL,
-    node text NOT NULL,
-    PRIMARY KEY (server_host, username, resource)
-);
-
-CREATE INDEX i_carboncopy_sh_user ON carboncopy USING btree (server_host, username);
-
 CREATE TABLE proxy65 (
     sid text NOT NULL,
     pid_t text NOT NULL,

sql/pg.sql

@@ -377,16 +377,6 @@ CREATE TABLE bosh (
 
 CREATE UNIQUE INDEX i_bosh_sid ON bosh USING btree (sid);
 
-CREATE TABLE carboncopy (
-    username text NOT NULL,
-    resource text NOT NULL,
-    namespace text NOT NULL,
-    node text NOT NULL
-);
-
-CREATE UNIQUE INDEX i_carboncopy_ur ON carboncopy USING btree (username, resource);
-CREATE INDEX i_carboncopy_user ON carboncopy USING btree (username);
-
 CREATE TABLE proxy65 (
     sid text NOT NULL,
     pid_t text NOT NULL,

src/ejabberd.erl

@@ -38,7 +38,7 @@
 -protocol({xep, 270, '1.0'}).
 
 -export([start/0, stop/0, halt/0, start_app/1, start_app/2,
-	 get_pid_file/0, check_app/1, module_name/1]).
+	 get_pid_file/0, check_app/1, module_name/1, is_loaded/0]).
 
 -include("logger.hrl").
 

src/ejabberd_acme.erl

@@ -1112,7 +1112,7 @@ save_certificate({ok, DomainName, Cert}) ->
 	%% that there is no certificate saved if it cannot be added in
 	%% certificate persistent storage
 	write_cert(CertificateFile, Cert, DomainName),
-	ok = ejabberd_pkix:add_certfile(CertificateFile),
+	{ok, _} = ejabberd_pkix:add_certfile(CertificateFile),
 	DataCert = #data_cert{
 		      domain = DomainName,
 		      pem = Cert,

src/ejabberd_app.erl

@@ -55,6 +55,7 @@ start(normal, _Args) ->
 		    ejabberd_system_monitor:start(),
 		    register_elixir_config_hooks(),
 		    ejabberd_cluster:wait_for_sync(infinity),
+		    ejabberd_hooks:run(ejabberd_started, []),
 		    {T2, _} = statistics(wall_clock),
 		    ?INFO_MSG("ejabberd ~s is started in the node ~p in ~.2fs",
 			      [ejabberd_config:get_version(),
@@ -76,6 +77,7 @@ start(_, _) ->
 %% This function is called when an application is about to be stopped,
 %% before shutting down the processes of the application.
 prep_stop(State) ->
+    ejabberd_hooks:run(ejabberd_stopping, []),
     ejabberd_listener:stop_listeners(),
     ejabberd_sm:stop(),
     gen_mod:stop_modules(),
@@ -150,6 +152,7 @@ start_apps() ->
     crypto:start(),
     ejabberd:start_app(sasl),
     ejabberd:start_app(ssl),
+    ejabberd:start_app(pkix),
     ejabberd:start_app(p1_utils),
     ejabberd:start_app(fast_yaml),
     ejabberd:start_app(fast_tls),

src/ejabberd_auth.erl

@@ -41,7 +41,8 @@
 	 get_password_s/2, get_password_with_authmodule/2,
 	 user_exists/2, user_exists_in_other_modules/3,
 	 remove_user/2, remove_user/3, plain_password_required/1,
-	 store_type/1, entropy/1, backend_type/1, password_format/1]).
+	 store_type/1, entropy/1, backend_type/1, password_format/1,
+	 which_users_exists/1]).
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
 	 terminate/2, code_change/3]).
@@ -411,6 +412,47 @@ user_exists_in_other_modules_loop([AuthModule | AuthModules], User, Server) ->
 	    maybe
     end.
 
+-spec which_users_exists(list({binary(), binary()})) -> list({binary(), binary()}).
+which_users_exists(USPairs) ->
+    ByServer = lists:foldl(
+	fun({User, Server}, Dict) ->
+	    LServer = jid:nameprep(Server),
+	    LUser =  jid:nodeprep(User),
+	    case gb_trees:lookup(LServer, Dict) of
+		none ->
+		    gb_trees:insert(LServer, gb_sets:singleton(LUser), Dict);
+		{value, Set} ->
+		    gb_trees:update(LServer, gb_sets:add(LUser, Set), Dict)
+	    end
+	end, gb_trees:empty(), USPairs),
+    Set = lists:foldl(
+	fun({LServer, UsersSet}, Results) ->
+	    UsersList = gb_sets:to_list(UsersSet),
+	    lists:foldl(
+		fun(M, Results2) ->
+		    try M:which_users_exists(LServer, UsersList) of
+			{error, _} ->
+			    Results2;
+			Res ->
+			    gb_sets:union(
+				gb_sets:from_list([{U, LServer} || U <- Res]),
+				Results2)
+		    catch
+			_:undef ->
+			    lists:foldl(
+				fun(U, R2) ->
+				    case user_exists(U, LServer) of
+					true ->
+					    gb_sets:add({U, LServer}, R2);
+					_ ->
+					    R2
+				    end
+				end, Results2, UsersList)
+		    end
+		end, Results, auth_modules(LServer))
+	end, gb_sets:empty(), gb_trees:to_list(ByServer)),
+    gb_sets:to_list(Set).
+
 -spec remove_user(binary(), binary()) -> ok.
 remove_user(User, Server) ->
     case validate_credentials(User, Server) of

src/ejabberd_auth_sql.erl

@@ -35,7 +35,7 @@
 -export([start/1, stop/1, set_password/3, try_register/3,
 	 get_users/2, count_users/2, get_password/2,
 	 remove_user/2, store_type/1, plain_password_required/1,
-	 convert_to_scram/1, opt_type/1, export/1]).
+	 convert_to_scram/1, opt_type/1, export/1, which_users_exists/2]).
 
 -include("scram.hrl").
 -include("logger.hrl").
@@ -247,6 +247,32 @@ users_number(LServer, [{prefix, Prefix}])
 users_number(LServer, []) ->
     users_number(LServer).
 
+which_users_exists(LServer, LUsers) when length(LUsers) =< 100 ->
+    try ejabberd_sql:sql_query(
+        LServer,
+        ?SQL("select @(username)s from users where username in %(LUsers)ls")) of
+        {selected, Matching} ->
+            [U || {U} <- Matching];
+        {error, _} = E ->
+            E
+    catch _:B ->
+        {error, B}
+    end;
+which_users_exists(LServer, LUsers) ->
+    {First, Rest} = lists:split(100, LUsers),
+    case which_users_exists(LServer, First) of
+        {error, _} = E ->
+            E;
+        V ->
+            case which_users_exists(LServer, Rest) of
+                {error, _} = E2 ->
+                    E2;
+                V2 ->
+                    V ++ V2
+            end
+    end.
+
+
 convert_to_scram(Server) ->
     LServer = jid:nameprep(Server),
     if

src/ejabberd_c2s.erl

@@ -49,7 +49,7 @@
 -export([get_presence/1, set_presence/2, resend_presence/1, resend_presence/2,
 	 open_session/1, call/3, cast/2, send/2, close/1, close/2, stop/1,
 	 reply/2, copy_state/2, set_timeout/2, route/2,
-	 host_up/1, host_down/1]).
+	 host_up/1, host_down/1, send_ws_ping/1]).
 
 -include("xmpp.hrl").
 -include("logger.hrl").
@@ -137,6 +137,11 @@ send_error(#{lserver := LServer} = State, Pkt, Err) ->
 	{Pkt1, State1} -> xmpp_stream_in:send_error(State1, Pkt1, Err)
     end.
 
+-spec send_ws_ping(pid()) -> ok;
+		  (state()) -> state().
+send_ws_ping(Ref) ->
+    xmpp_stream_in:send_ws_ping(Ref).
+
 -spec route(pid(), term()) -> boolean().
 route(Pid, Term) ->
     ejabberd_cluster:send(Pid, Term).
@@ -633,7 +638,7 @@ route_probe_reply(From, #{jid := To,
     Subscription = get_subscription(To, From),
     if IsAnotherResource orelse
        Subscription == both orelse Subscription == from ->
-	    Packet = xmpp_util:add_delay_info(LastPres, To, TS),
+	    Packet = misc:add_delay_info(LastPres, To, TS),
 	    case privacy_check_packet(State, Packet, out) of
 		deny ->
 		    ok;
@@ -982,8 +987,8 @@ listen_opt_type(certfile = Opt) ->
     fun(S) ->
 	    ?WARNING_MSG("Listening option '~s' for ~s is deprecated, use "
 			 "'certfiles' global option instead", [Opt, ?MODULE]),
-	    ok = ejabberd_pkix:add_certfile(S),
-	    iolist_to_binary(S)
+	    {ok, File} = ejabberd_pkix:add_certfile(S),
+	    File
     end;
 listen_opt_type(starttls) -> fun(B) when is_boolean(B) -> B end;
 listen_opt_type(starttls_required) -> fun(B) when is_boolean(B) -> B end;

src/ejabberd_captcha.erl

@@ -128,10 +128,10 @@ create_captcha_x(SID, To, Lang, Limiter, #xdata{fields = Fs} = X) ->
 					  "visit the web page.">>),
 	  Imageurl = get_url(<<Id/binary, "/image">>),
 	  NewFs = [mk_field(hidden, <<"FORM_TYPE">>, ?NS_CAPTCHA)|Fs] ++
-		[#xdata_field{type = fixed, values = [HelpTxt]},
+		[#xdata_field{type = fixed, var = <<"captcha-fallback-text">>, values = [HelpTxt]},
 		 #xdata_field{type = hidden, var = <<"captchahidden">>,
 			      values = [<<"workaround-for-psi">>]},
-		 #xdata_field{type = 'text-single', var = <<"url">>,
+		 #xdata_field{type = 'text-single', var = <<"captcha-fallback-url">>,
 			      label = translate:translate(
 					Lang, <<"CAPTCHA web page">>),
 			      values = [Imageurl]},

src/ejabberd_http.erl

@@ -69,7 +69,8 @@
 		default_host,
 		custom_headers,
 		trail = <<>>,
-		addr_re
+		addr_re,
+		sock_peer_name = none
 	       }).
 
 -define(XHTML_DOCTYPE,
@@ -143,6 +144,7 @@ init({SockMod, Socket}, Opts) ->
 		 true -> [{[], ejabberd_xmlrpc}];
 		 false -> []
 	     end,
+    SockPeer =  proplists:get_value(sock_peer_name, Opts, none),
     DefinedHandlers = proplists:get_value(request_handlers, Opts, []),
     RequestHandlers = DefinedHandlers ++ Captcha ++ Register ++
         Admin ++ Bind ++ XMLRPC,
@@ -159,6 +161,7 @@ init({SockMod, Socket}, Opts) ->
 		   custom_headers = CustomHeaders,
 		   options = Opts,
 		   request_handlers = RequestHandlers,
+		   sock_peer_name = SockPeer,
 		   addr_re = RE},
     try receive_headers(State) of
         V -> V
@@ -411,11 +414,11 @@ extract_path_query(#state{request_method = Method,
     when Method =:= 'GET' orelse
 	   Method =:= 'HEAD' orelse
 	     Method =:= 'DELETE' orelse Method =:= 'OPTIONS' ->
-    case catch url_decode_q_split(Path) of
-	{'EXIT', _} -> {State, false};
-	{NPath, Query} ->
-	    LPath = normalize_path([NPE
-				    || NPE <- str:tokens(path_decode(NPath), <<"/">>)]),
+    case catch url_decode_q_split_normalize(Path) of
+	{'EXIT', Error} ->
+	    ?DEBUG("Error decoding URL '~p': ~p", [Path, Error]),
+	    {State, false};
+	{LPath, Query} ->
 	    LQuery = case catch parse_urlencoded(Query) of
 			 {'EXIT', _Reason} -> [];
 			 LQ -> LQ
@@ -429,11 +432,11 @@ extract_path_query(#state{request_method = Method,
 			  sockmod = _SockMod,
 			  socket = _Socket} = State)
   when (Method =:= 'POST' orelse Method =:= 'PUT') andalso Len>0 ->
-    case catch url_decode_q_split(Path) of
-        {'EXIT', _} -> {State, false};
-        {NPath, _Query} ->
-            LPath = normalize_path(
-		      [NPE || NPE <- str:tokens(path_decode(NPath), <<"/">>)]),
+    case catch url_decode_q_split_normalize(Path) of
+	{'EXIT', Error} ->
+	    ?DEBUG("Error decoding URL '~p': ~p", [Path, Error]),
+	    {State, false};
+        {LPath, _Query} ->
 	    case Method of
 		'PUT' ->
 		    {State, {LPath, [], Trail}};
@@ -463,6 +466,7 @@ process_request(#state{request_method = Method,
 		       request_version = Version,
 		       sockmod = SockMod,
 		       socket = Socket,
+		       sock_peer_name = SockPeer,
 		       options = Options,
 		       request_host = Host,
 		       request_port = Port,
@@ -481,13 +485,17 @@ process_request(#state{request_method = Method,
 	{State2, false} ->
 	    {State2, make_bad_request(State)};
 	{State2, {LPath, LQuery, Data}} ->
-	    PeerName =
-		case SockMod of
-		    gen_tcp ->
-			inet:peername(Socket);
-		    _ ->
-			SockMod:peername(Socket)
-		end,
+	    PeerName = case SockPeer of
+			   none ->
+			       case SockMod of
+				   gen_tcp ->
+				       inet:peername(Socket);
+				   _ ->
+				       SockMod:peername(Socket)
+			       end;
+			   {_, Peer} ->
+			       {ok, Peer}
+		       end,
             IPHere = case PeerName of
                          {ok, V} -> V;
                          {error, _} = E -> throw(E)
@@ -573,7 +581,7 @@ is_ipchain_trusted(UserIPs, Masks) ->
 		    lists:any(
 			fun({Mask, MaskLen}) ->
 			    acl:ip_matches_mask(IP2, Mask, MaskLen)
-			end, [{{127,0,0,1}, 8} | Masks]);
+			end, Masks);
 		_ ->
 		    false
 	    end
@@ -724,6 +732,12 @@ file_format_error(Reason) ->
 	Text -> Text
     end.
 
+url_decode_q_split_normalize(Path) ->
+    {NPath, Query} = url_decode_q_split(Path),
+    LPath = normalize_path([NPE
+		    || NPE <- str:tokens(path_decode(NPath), <<"/">>)]),
+    {LPath, Query}.
+
 % Code below is taken (with some modifications) from the yaws webserver, which
 % is distributed under the following license:
 %
@@ -965,8 +979,8 @@ listen_opt_type(certfile = Opt) ->
     fun(S) ->
 	    ?WARNING_MSG("Listening option '~s' for ~s is deprecated, use "
 			 "'certfiles' global option instead", [Opt, ?MODULE]),
-	    ok = ejabberd_pkix:add_certfile(S),
-	    iolist_to_binary(S)
+	    {ok, File} = ejabberd_pkix:add_certfile(S),
+	    File
     end;
 listen_opt_type(captcha) ->
     fun(B) when is_boolean(B) -> B end;

src/ejabberd_listener.erl

@@ -33,7 +33,7 @@
 	 start_listeners/0, start_listener/3, stop_listeners/0,
 	 stop_listener/2, add_listener/3, delete_listener/2,
 	 transform_options/1, validate_cfg/1, opt_type/1,
-	 config_reloaded/0]).
+	 config_reloaded/0, get_certfiles/0]).
 %% Legacy API
 -export([parse_listener_portip/2]).
 
@@ -63,12 +63,7 @@ init(_) ->
     ets:new(?MODULE, [named_table, public]),
     ejabberd_hooks:add(config_reloaded, ?MODULE, config_reloaded, 50),
     Listeners = ejabberd_config:get_option(listen, []),
-    case add_certfiles(Listeners) of
-	ok ->
-	    {ok, {{one_for_one, 10, 1}, listeners_childspec(Listeners)}};
-	{error, _} = Err ->
-	    Err
-    end.
+    {ok, {{one_for_one, 10, 1}, listeners_childspec(Listeners)}}.
 
 -spec listeners_childspec([listener()]) -> [supervisor:child_spec()].
 listeners_childspec(Listeners) ->
@@ -209,26 +204,49 @@ accept(ListenSocket, Module, Opts, Sup, Interval) ->
     NewInterval = check_rate_limit(Interval),
     case gen_tcp:accept(ListenSocket) of
 	{ok, Socket} ->
-	    case {inet:sockname(Socket), inet:peername(Socket)} of
-		{{ok, {Addr, Port}}, {ok, {PAddr, PPort}}} ->
-		    Receiver = case start_connection(Module, Socket, Opts, Sup) of
-				   {ok, RecvPid} ->
-				       RecvPid;
-				   _ ->
-				       gen_tcp:close(Socket),
-				       none
-			       end,
-		    ?INFO_MSG("(~p) Accepted connection ~s:~p -> ~s:~p",
-			      [Receiver,
-			       ejabberd_config:may_hide_data(inet_parse:ntoa(PAddr)),
-			       PPort, inet_parse:ntoa(Addr), Port]);
+	    case proplists:get_value(use_proxy_protocol, Opts, false) of
+		true ->
+		    case proxy_protocol:decode(gen_tcp, Socket, 10000) of
+			{error, Err} ->
+			    ?ERROR_MSG("(~w) Proxy protocol parsing failed: ~s",
+				       [ListenSocket, inet:format_error(Err)]),
+			    gen_tcp:close(Socket);
+			{{Addr, Port}, {PAddr, PPort}} = SP ->
+			    Opts2 = [{sock_peer_name, SP} | Opts],
+			    Receiver = case start_connection(Module, Socket, Opts2, Sup) of
+					   {ok, RecvPid} ->
+					       RecvPid;
+					   _ ->
+					       gen_tcp:close(Socket),
+					       none
+				       end,
+			    ?INFO_MSG("(~p) Accepted proxied connection ~s:~p -> ~s:~p",
+				      [Receiver,
+				       ejabberd_config:may_hide_data(inet_parse:ntoa(PAddr)),
+				       PPort, inet_parse:ntoa(Addr), Port])
+		    end;
 		_ ->
-		    gen_tcp:close(Socket)
+		    case {inet:sockname(Socket), inet:peername(Socket)} of
+			{{ok, {Addr, Port}}, {ok, {PAddr, PPort}}} ->
+			    Receiver = case start_connection(Module, Socket, Opts, Sup) of
+					   {ok, RecvPid} ->
+					       RecvPid;
+					   _ ->
+					       gen_tcp:close(Socket),
+					       none
+				       end,
+			    ?INFO_MSG("(~p) Accepted connection ~s:~p -> ~s:~p",
+				      [Receiver,
+				       ejabberd_config:may_hide_data(inet_parse:ntoa(PAddr)),
+				       PPort, inet_parse:ntoa(Addr), Port]);
+			_ ->
+			    gen_tcp:close(Socket)
+		    end
 	    end,
 	    accept(ListenSocket, Module, Opts, Sup, NewInterval);
 	{error, Reason} ->
 	    ?ERROR_MSG("(~w) Failed TCP accept: ~s",
-                       [ListenSocket, inet:format_error(Reason)]),
+		       [ListenSocket, inet:format_error(Reason)]),
 	    accept(ListenSocket, Module, Opts, Sup, NewInterval)
     end.
 
@@ -384,6 +402,16 @@ config_reloaded() ->
 	      end
       end, New).
 
+-spec get_certfiles() -> [binary()].
+get_certfiles() ->
+    lists:filtermap(
+      fun({_, _, Opts}) ->
+	      case proplists:get_value(certfile, Opts) of
+		  undefined -> false;
+		  Cert -> {true, Cert}
+	      end
+      end, ets:tab2list(?MODULE)).
+
 -spec report_socket_error(inet:posix(), endpoint(), module()) -> ok.
 report_socket_error(Reason, EndPoint, Module) ->
     ?ERROR_MSG("Failed to open socket at ~s for ~s: ~s",
@@ -432,20 +460,6 @@ check_rate_limit(Interval) ->
     end,
     NewInterval.
 
--spec add_certfiles([listener()]) -> ok | {error, any()}.
-add_certfiles([{_, _, Opts}|Listeners]) ->
-    case lists:keyfind(certfile, 1, Opts) of
-	{_, Path} ->
-	    case ejabberd_pkix:add_certfile(Path) of
-		ok -> add_certfiles(Listeners);
-		{error, _} = Err -> Err
-	    end;
-	false ->
-	    add_certfiles(Listeners)
-    end;
-add_certfiles([]) ->
-    ok.
-
 transform_option({{Port, IP, Transport}, Mod, Opts}) ->
     IPStr = if is_tuple(IP) ->
                     list_to_binary(inet_parse:ntoa(IP));
@@ -652,12 +666,12 @@ listen_opt_type(supervisor) ->
     fun(B) when is_boolean(B) -> B end;
 listen_opt_type(certfile) ->
     fun(S) ->
-	    ok = ejabberd_pkix:add_certfile(S),
-	    iolist_to_binary(S)
+	    {ok, File} = ejabberd_pkix:add_certfile(S),
+	    File
     end;
 listen_opt_type(ciphers) -> fun iolist_to_binary/1;
 listen_opt_type(dhfile) -> fun misc:try_read_file/1;
-listen_opt_type(cafile) -> fun misc:try_read_file/1;
+listen_opt_type(cafile) -> fun ejabberd_pkix:try_certfile/1;
 listen_opt_type(protocol_options) ->
     fun (Options) -> str:join(Options, <<"|">>) end;
 listen_opt_type(tls_compression) ->
@@ -674,7 +688,9 @@ listen_opt_type(max_fsm_queue) ->
 listen_opt_type(shaper) ->
     fun acl:shaper_rules_validator/1;
 listen_opt_type(access) ->
-    fun acl:access_rules_validator/1.
+    fun acl:access_rules_validator/1;
+listen_opt_type(use_proxy_protocol) ->
+    fun(B) when is_boolean(B) -> B end.
 
 listen_options() ->
     [module, port,
@@ -684,6 +700,7 @@ listen_options() ->
      {inet6, false},
      {accept_interval, 0},
      {backlog, 5},
+     {use_proxy_protocol, false},
      {supervisor, true}].
 
 opt_type(listen) -> fun validate_cfg/1;

src/ejabberd_logger.erl

@@ -220,7 +220,7 @@ set(LogLevel) when is_integer(LogLevel) ->
     end,
     case LogLevel of
 	5 -> xmpp:set_config([{debug, true}]);
-	_ -> ok
+	_ -> xmpp:set_config([{debug, false}])
     end,
     {module, lager};
 set({_LogLevel, _}) ->

src/ejabberd_regexp.erl

@@ -36,7 +36,7 @@ exec({ReM, ReF, ReA}, {RgM, RgF, RgA}) ->
 -spec run(binary(), binary()) -> match | nomatch | {error, any()}.
 
 run(String, Regexp) ->
-    case exec({re, run, [String, Regexp, [{capture, none}]]},
+    case exec({re, run, [String, Regexp, [{capture, none}, unicode]]},
 	      {regexp, first_match, [binary_to_list(String),
                                      binary_to_list(Regexp)]})
 	of

src/ejabberd_s2s_in.erl

@@ -135,16 +135,16 @@ process_closed(#{server := LServer} = State, Reason) ->
 %%%===================================================================
 %%% xmpp_stream_in callbacks
 %%%===================================================================
-tls_options(#{tls_options := TLSOpts, server_host := LServer}) ->
+tls_options(#{tls_options := TLSOpts, lserver := LServer}) ->
     ejabberd_s2s:tls_options(LServer, TLSOpts).
 
-tls_required(#{server_host := LServer}) ->
+tls_required(#{lserver := LServer}) ->
     ejabberd_s2s:tls_required(LServer).
 
-tls_enabled(#{server_host := LServer}) ->
+tls_enabled(#{lserver := LServer}) ->
     ejabberd_s2s:tls_enabled(LServer).
 
-compress_methods(#{server_host := LServer}) ->
+compress_methods(#{lserver := LServer}) ->
     case ejabberd_s2s:zlib_enabled(LServer) of
 	true -> [<<"zlib">>];
 	false -> []
@@ -181,14 +181,14 @@ handle_auth_success(RServer, Mech, _AuthModule,
     ?INFO_MSG("(~s) Accepted inbound s2s ~s authentication ~s -> ~s (~s)",
 	      [xmpp_socket:pp(Socket), Mech, RServer, LServer,
 	       ejabberd_config:may_hide_data(misc:ip_to_list(IP))]),
-    State1 = case ejabberd_s2s:allow_host(ServerHost, RServer) of
+    State1 = case ejabberd_s2s:allow_host(LServer, RServer) of
 		 true ->
 		     AuthDomains1 = sets:add_element(RServer, AuthDomains),
 		     State0 = change_shaper(State, RServer),
 		     State0#{auth_domains => AuthDomains1};
 		 false ->
 		     State
-	   end,
+	     end,
     ejabberd_hooks:run_fold(s2s_in_auth_result, ServerHost, State1, [true, RServer]).
 
 handle_auth_failure(RServer, Mech, Reason,
@@ -221,7 +221,7 @@ handle_authenticated_packet(Pkt0, #{ip := {IP, _}} = State) ->
 	    case Pkt1 of
 		drop -> ok;
 		_ -> ejabberd_router:route(Pkt1)
-    end,
+	    end,
 	    State2;
 	{error, Err} ->
 	    send(State, Err)
@@ -249,9 +249,9 @@ init([State, Opts]) ->
 		    (_) -> false
 		 end, Opts),
     TLSOpts2 = case proplists:get_bool(tls_compression, Opts) of
-                   false -> [compression_none | TLSOpts1];
-                   true -> TLSOpts1
-    end,
+		   false -> [compression_none | TLSOpts1];
+		   true -> TLSOpts1
+	       end,
     Timeout = ejabberd_config:negotiation_timeout(),
     State1 = State#{tls_options => TLSOpts2,
 		    auth_domains => sets:new(),
@@ -327,7 +327,7 @@ check_to(#jid{lserver = LServer}, _State) ->
     ejabberd_router:is_my_route(LServer).
 
 -spec set_idle_timeout(state()) -> state().
-set_idle_timeout(#{server_host := LServer,
+set_idle_timeout(#{lserver := LServer,
 		   established := true} = State) ->
     Timeout = ejabberd_s2s:get_idle_timeout(LServer),
     xmpp_stream_in:set_timeout(State, Timeout);
@@ -344,8 +344,8 @@ listen_opt_type(certfile = Opt) ->
     fun(S) ->
 	    ?WARNING_MSG("Listening option '~s' for ~s is deprecated, use "
 			 "'certfiles' global option instead", [Opt, ?MODULE]),
-	    ok = ejabberd_pkix:add_certfile(S),
-	    iolist_to_binary(S)
+	    {ok, File} = ejabberd_pkix:add_certfile(S),
+	    File
     end.
 
 listen_options() ->

src/ejabberd_sm.erl

@@ -50,8 +50,6 @@
 	 set_presence/7,
 	 unset_presence/6,
 	 close_session_unset_presence/5,
-	 set_offline_info/5,
-	 get_offline_info/4,
 	 dirty_get_sessions_list/0,
 	 dirty_get_my_sessions_list/0,
 	 get_vh_session_list/1,
@@ -68,6 +66,8 @@
 	 get_session_sids/2,
 	 get_user_info/2,
 	 get_user_info/3,
+	 set_user_info/5,
+	 del_user_info/4,
 	 get_user_ip/3,
 	 get_max_user_sessions/2,
 	 get_all_pids/0,
@@ -78,8 +78,7 @@
 	 host_down/1,
 	 make_sid/0,
 	 clean_cache/1,
-	 config_reloaded/0,
-	 is_online/1
+	 config_reloaded/0
 	]).
 
 -export([init/1, handle_call/3, handle_cast/2,
@@ -211,14 +210,14 @@ get_user_resources(User, Server) ->
     LUser = jid:nodeprep(User),
     LServer = jid:nameprep(Server),
     Mod = get_sm_backend(LServer),
-    Ss = online(get_sessions(Mod, LUser, LServer)),
+    Ss = get_sessions(Mod, LUser, LServer),
     [element(3, S#session.usr) || S <- clean_session_list(Ss)].
 
 -spec get_user_present_resources(binary(), binary()) -> [tuple()].
 
 get_user_present_resources(LUser, LServer) ->
     Mod = get_sm_backend(LServer),
-    Ss = online(get_sessions(Mod, LUser, LServer)),
+    Ss = get_sessions(Mod, LUser, LServer),
     [{S#session.priority, element(3, S#session.usr)}
      || S <- clean_session_list(Ss), is_integer(S#session.priority)].
 
@@ -229,7 +228,7 @@ get_user_ip(User, Server, Resource) ->
     LServer = jid:nameprep(Server),
     LResource = jid:resourceprep(Resource),
     Mod = get_sm_backend(LServer),
-    case online(get_sessions(Mod, LUser, LServer, LResource)) of
+    case get_sessions(Mod, LUser, LServer, LResource) of
 	[] ->
 	    undefined;
 	Ss ->
@@ -242,7 +241,7 @@ get_user_info(User, Server) ->
     LUser = jid:nodeprep(User),
     LServer = jid:nameprep(Server),
     Mod = get_sm_backend(LServer),
-    Ss = online(get_sessions(Mod, LUser, LServer)),
+    Ss = get_sessions(Mod, LUser, LServer),
     [{LResource, [{node, node(Pid)}, {ts, Ts}, {pid, Pid},
 		  {priority, Priority} | Info]}
      || #session{usr = {_, _, LResource},
@@ -257,7 +256,7 @@ get_user_info(User, Server, Resource) ->
     LServer = jid:nameprep(Server),
     LResource = jid:resourceprep(Resource),
     Mod = get_sm_backend(LServer),
-    case online(get_sessions(Mod, LUser, LServer, LResource)) of
+    case get_sessions(Mod, LUser, LServer, LResource) of
 	[] ->
 	    offline;
 	Ss ->
@@ -269,6 +268,44 @@ get_user_info(User, Server, Resource) ->
 	     |Session#session.info]
     end.
 
+-spec set_user_info(binary(), binary(), binary(), atom(), term()) -> ok | {error, any()}.
+set_user_info(User, Server, Resource, Key, Val) ->
+    LUser = jid:nodeprep(User),
+    LServer = jid:nameprep(Server),
+    LResource = jid:resourceprep(Resource),
+    Mod = get_sm_backend(LServer),
+    case get_sessions(Mod, LUser, LServer, LResource) of
+	[] -> {error, notfound};
+	Ss ->
+	    lists:foldl(
+	      fun(#session{sid = {_, Pid},
+			   info = Info} = Session, _) when Pid == self() ->
+		      Info1 = lists:keystore(Key, 1, Info, {Key, Val}),
+		      set_session(Session#session{info = Info1});
+		 (_, Acc) ->
+		      Acc
+	      end, {error, not_owner}, Ss)
+    end.
+
+-spec del_user_info(binary(), binary(), binary(), atom()) -> ok | {error, any()}.
+del_user_info(User, Server, Resource, Key) ->
+    LUser = jid:nodeprep(User),
+    LServer = jid:nameprep(Server),
+    LResource = jid:resourceprep(Resource),
+    Mod = get_sm_backend(LServer),
+    case get_sessions(Mod, LUser, LServer, LResource) of
+	[] -> {error, notfound};
+	Ss ->
+	    lists:foldl(
+	      fun(#session{sid = {_, Pid},
+			   info = Info} = Session, _) when Pid == self() ->
+		      Info1 = lists:keydelete(Key, 1, Info),
+		      set_session(Session#session{info = Info1});
+		 (_, Acc) ->
+		      Acc
+	      end, {error, not_owner}, Ss)
+    end.
+
 -spec set_presence(sid(), binary(), binary(), binary(),
                    prio(), presence(), info()) -> ok.
 
@@ -316,7 +353,7 @@ get_session_sid(User, Server, Resource) ->
     LServer = jid:nameprep(Server),
     LResource = jid:resourceprep(Resource),
     Mod = get_sm_backend(LServer),
-    case online(get_sessions(Mod, LUser, LServer, LResource)) of
+    case get_sessions(Mod, LUser, LServer, LResource) of
 	[] ->
 	    none;
 	Ss ->
@@ -330,43 +367,15 @@ get_session_sids(User, Server) ->
     LUser = jid:nodeprep(User),
     LServer = jid:nameprep(Server),
     Mod = get_sm_backend(LServer),
-    Sessions = online(get_sessions(Mod, LUser, LServer)),
+    Sessions = get_sessions(Mod, LUser, LServer),
     [SID || #session{sid = SID} <- Sessions].
 
--spec set_offline_info(sid(), binary(), binary(), binary(), info()) -> ok.
-
-set_offline_info(SID, User, Server, Resource, Info) ->
-    LUser = jid:nodeprep(User),
-    LServer = jid:nameprep(Server),
-    LResource = jid:resourceprep(Resource),
-    set_session(SID, LUser, LServer, LResource, undefined, [offline | Info]).
-
--spec get_offline_info(erlang:timestamp(), binary(), binary(),
-                       binary()) -> none | info().
-
-get_offline_info(Time, User, Server, Resource) ->
-    LUser = jid:nodeprep(User),
-    LServer = jid:nameprep(Server),
-    LResource = jid:resourceprep(Resource),
-    Mod = get_sm_backend(LServer),
-    case get_sessions(Mod, LUser, LServer, LResource) of
-	[#session{sid = {Time, _}, info = Info}] ->
-	    case proplists:get_bool(offline, Info) of
-		true ->
-		    Info;
-		false ->
-		    none
-	    end;
-	_ ->
-	    none
-    end.
-
 -spec dirty_get_sessions_list() -> [ljid()].
 
 dirty_get_sessions_list() ->
     lists:flatmap(
       fun(Mod) ->
-	      [S#session.usr || S <- online(get_sessions(Mod))]
+	      [S#session.usr || S <- get_sessions(Mod)]
       end, get_sm_backends()).
 
 -spec dirty_get_my_sessions_list() -> [#session{}].
@@ -374,7 +383,7 @@ dirty_get_sessions_list() ->
 dirty_get_my_sessions_list() ->
     lists:flatmap(
       fun(Mod) ->
-	      [S || S <- online(get_sessions(Mod)),
+	      [S || S <- get_sessions(Mod),
 		    node(element(2, S#session.sid)) == node()]
       end, get_sm_backends()).
 
@@ -383,14 +392,14 @@ dirty_get_my_sessions_list() ->
 get_vh_session_list(Server) ->
     LServer = jid:nameprep(Server),
     Mod = get_sm_backend(LServer),
-    [S#session.usr || S <- online(get_sessions(Mod, LServer))].
+    [S#session.usr || S <- get_sessions(Mod, LServer)].
 
 -spec get_all_pids() -> [pid()].
 
 get_all_pids() ->
     lists:flatmap(
       fun(Mod) ->
-	      [element(2, S#session.sid) || S <- online(get_sessions(Mod))]
+	      [element(2, S#session.sid) || S <- get_sessions(Mod)]
       end, get_sm_backends()).
 
 -spec get_vh_session_number(binary()) -> non_neg_integer().
@@ -398,7 +407,7 @@ get_all_pids() ->
 get_vh_session_number(Server) ->
     LServer = jid:nameprep(Server),
     Mod = get_sm_backend(LServer),
-    length(online(get_sessions(Mod, LServer))).
+    length(get_sessions(Mod, LServer)).
 
 %% Why the hell do we have so many similar kicks?
 c2s_handle_info(#{lang := Lang} = State, replaced) ->
@@ -514,9 +523,13 @@ set_session(SID, User, Server, Resource, Priority, Info) ->
     LResource = jid:resourceprep(Resource),
     US = {LUser, LServer},
     USR = {LUser, LServer, LResource},
+    set_session(#session{sid = SID, usr = USR, us = US,
+			 priority = Priority, info = Info}).
+
+-spec set_session(#session{}) -> ok | {error, any()}.
+set_session(#session{us = {LUser, LServer}} = Session) ->
     Mod = get_sm_backend(LServer),
-    case Mod:set_session(#session{sid = SID, usr = USR, us = US,
-				  priority = Priority, info = Info}) of
+    case Mod:set_session(Session) of
 	ok ->
 	    case use_cache(Mod, LServer) of
 		true ->
@@ -579,16 +592,6 @@ delete_session(Mod, #session{usr = {LUser, LServer, _}} = Session) ->
 	    ok
     end.
 
--spec online([#session{}]) -> [#session{}].
-
-online(Sessions) ->
-    lists:filter(fun is_online/1, Sessions).
-
--spec is_online(#session{}) -> boolean().
-
-is_online(#session{info = Info}) ->
-    not proplists:get_bool(offline, Info).
-
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 -spec do_route(jid(), term()) -> any().
 do_route(#jid{lresource = <<"">>} = To, Term) ->
@@ -600,7 +603,7 @@ do_route(To, Term) ->
     ?DEBUG("broadcasting ~p to ~s", [Term, jid:encode(To)]),
     {U, S, R} = jid:tolower(To),
     Mod = get_sm_backend(S),
-    case online(get_sessions(Mod, U, S, R)) of
+    case get_sessions(Mod, U, S, R) of
 	[] ->
 	    ?DEBUG("dropping broadcast to unavailable resourse: ~p", [Term]);
 	Ss ->
@@ -631,7 +634,7 @@ do_route(#presence{to = To, type = T} = Packet)
 		      ejabberd_c2s:route(Pid, {route, Packet1});
 		 (_) ->
 		      ok
-	      end, online(get_sessions(Mod, LUser, LServer)));
+	      end, get_sessions(Mod, LUser, LServer));
 	false ->
 	    ok
     end;
@@ -660,7 +663,7 @@ do_route(Packet) ->
     To = xmpp:get_to(Packet),
     {LUser, LServer, LResource} = jid:tolower(To),
     Mod = get_sm_backend(LServer),
-    case online(get_sessions(Mod, LUser, LServer, LResource)) of
+    case get_sessions(Mod, LUser, LServer, LResource) of
 	[] ->
 	    case Packet of
 		#message{type = T} when T == chat; T == normal ->
@@ -708,8 +711,8 @@ route_message(#message{to = To, type = Type} = Packet) ->
 					  (P >= 0) and (Type == headline) ->
 				LResource = jid:resourceprep(R),
 				Mod = get_sm_backend(LServer),
-				case online(get_sessions(Mod, LUser, LServer,
-							     LResource)) of
+				case get_sessions(Mod, LUser, LServer,
+						  LResource) of
 				  [] ->
 				      ok; % Race condition
 				  Ss ->
@@ -780,13 +783,9 @@ check_for_sessions_to_replace(User, Server, Resource) ->
 check_existing_resources(LUser, LServer, LResource) ->
     Mod = get_sm_backend(LServer),
     Ss = get_sessions(Mod, LUser, LServer, LResource),
-    {OnlineSs, OfflineSs} = lists:partition(fun is_online/1, Ss),
-    lists:foreach(fun(S) ->
-			  delete_session(Mod, S)
-		  end, OfflineSs),
-    if OnlineSs == [] -> ok;
+    if Ss == [] -> ok;
        true ->
-	   SIDs = [SID || #session{sid = SID} <- OnlineSs],
+	   SIDs = [SID || #session{sid = SID} <- Ss],
 	   MaxSID = lists:max(SIDs),
 	   lists:foreach(fun ({_, Pid} = S) when S /= MaxSID ->
 				 ejabberd_c2s:route(Pid, replaced);
@@ -806,22 +805,17 @@ get_resource_sessions(User, Server, Resource) ->
     LServer = jid:nameprep(Server),
     LResource = jid:resourceprep(Resource),
     Mod = get_sm_backend(LServer),
-    [S#session.sid || S <- online(get_sessions(Mod, LUser, LServer, LResource))].
+    [S#session.sid || S <- get_sessions(Mod, LUser, LServer, LResource)].
 
 -spec check_max_sessions(binary(), binary()) -> ok | replaced.
 check_max_sessions(LUser, LServer) ->
     Mod = get_sm_backend(LServer),
     Ss = get_sessions(Mod, LUser, LServer),
-    {OnlineSs, OfflineSs} = lists:partition(fun is_online/1, Ss),
     MaxSessions = get_max_user_sessions(LUser, LServer),
-    if length(OnlineSs) =< MaxSessions -> ok;
+    if length(Ss) =< MaxSessions -> ok;
        true ->
-	    #session{sid = {_, Pid}} = lists:min(OnlineSs),
+	    #session{sid = {_, Pid}} = lists:min(Ss),
 	    ejabberd_c2s:route(Pid, replaced)
-    end,
-    if length(OfflineSs) =< MaxSessions -> ok;
-       true ->
-	    delete_session(Mod, lists:min(OfflineSs))
     end.
 
 %% Get the user_max_session setting
@@ -843,7 +837,7 @@ get_max_user_sessions(LUser, Host) ->
 
 force_update_presence({LUser, LServer}) ->
     Mod = get_sm_backend(LServer),
-    Ss = online(get_sessions(Mod, LUser, LServer)),
+    Ss = get_sessions(Mod, LUser, LServer),
     lists:foreach(fun (#session{sid = {_, Pid}}) ->
 			  ejabberd_c2s:resend_presence(Pid)
 		  end,

src/ejabberd_sql.erl

@@ -37,12 +37,12 @@
 	 sql_query_t/1,
 	 sql_transaction/2,
 	 sql_bloc/2,
-         abort/1,
-         restart/1,
-         use_new_schema/0,
-         sql_query_to_iolist/1,
+	 abort/1,
+	 restart/1,
+	 use_new_schema/0,
+	 sql_query_to_iolist/1,
 	 escape/1,
-         standard_escape/1,
+	 standard_escape/1,
 	 escape_like/1,
 	 escape_like_arg/1,
 	 escape_like_arg_circumflex/1,
@@ -55,7 +55,8 @@
 	 freetds_config/0,
 	 odbcinst_config/0,
 	 init_mssql/1,
-	 keep_alive/2]).
+	 keep_alive/2,
+	 to_list/2]).
 
 %% gen_fsm callbacks
 -export([init/1, handle_event/3, handle_sync_event/4,
@@ -176,7 +177,7 @@ keep_alive(Host, PID) ->
 		    {sql_cmd, {sql_query, ?KEEPALIVE_QUERY},
 		     p1_time_compat:monotonic_time(milli_seconds)},
 		    query_timeout(Host)) of
-	{selected,[<<"1">>],[[<<"1">>]]} ->
+	{selected,_,[[<<"1">>]]} ->
 	    ok;
 	_Err ->
 	    ?ERROR_MSG("keep alive query failed, closing connection: ~p", [_Err]),
@@ -258,6 +259,10 @@ to_bool(true) -> true;
 to_bool(1) -> true;
 to_bool(_) -> false.
 
+to_list(EscapeFun, Val) ->
+    Escaped = lists:join(<<",">>, lists:map(EscapeFun, Val)),
+    [<<"(">>, Escaped, <<")">>].
+
 encode_term(Term) ->
     escape(list_to_binary(
              erl_prettypr:format(erl_syntax:abstract(Term),
@@ -1162,7 +1167,7 @@ opt_type(sql_username) -> fun iolist_to_binary/1;
 opt_type(sql_ssl) -> fun(B) when is_boolean(B) -> B end;
 opt_type(sql_ssl_verify) -> fun(B) when is_boolean(B) -> B end;
 opt_type(sql_ssl_certfile) -> fun ejabberd_pkix:try_certfile/1;
-opt_type(sql_ssl_cafile) -> fun misc:try_read_file/1;
+opt_type(sql_ssl_cafile) -> fun ejabberd_pkix:try_certfile/1;
 opt_type(sql_query_timeout) ->
     fun (I) when is_integer(I), I > 0 -> I end;
 opt_type(sql_connect_timeout) ->

src/ejabberd_sql_pt.erl

@@ -306,6 +306,20 @@ parse1([$%, $( | S], Acc, State) ->
                     false ->
                         append_string("0=0", State3)
                 end;
+            {list, InternalType} ->
+                Convert = erl_syntax:application(
+                    erl_syntax:atom(ejabberd_sql),
+                    erl_syntax:atom(to_list),
+                    [erl_syntax:record_access(
+                        erl_syntax:variable(?ESCAPE_VAR),
+                        erl_syntax:atom(?ESCAPE_RECORD),
+                        erl_syntax:atom(InternalType)),
+                     erl_syntax:variable(Name)]),
+                State2#state{'query' = [{var, Var} | State2#state.'query'],
+                             args = [Convert | State2#state.args],
+                             params = [Var | State2#state.params],
+                             param_pos = State2#state.param_pos + 1,
+                             used_vars = [Name | State2#state.used_vars]};
             _ ->
                 Convert =
                     erl_syntax:application(
@@ -335,6 +349,19 @@ parse_name(S, IsArg, State) ->
 parse_name([], _Acc, _Depth, _IsArg, State) ->
     throw({error, State#state.loc,
            "expected ')', found end of string"});
+parse_name([$), $l, T | S], Acc, 0, true, State) ->
+    Type = case T of
+               $d -> {list, integer};
+               $s -> {list, string};
+               $b -> {list, boolean};
+               _ ->
+                   throw({error, State#state.loc,
+                          ["unknown type specifier 'l", T, "'"]})
+           end,
+    {lists:reverse(Acc), Type, S, State};
+parse_name([$), $l, T | _], _Acc, 0, false, State) ->
+    throw({error, State#state.loc,
+           ["list type 'l", T, "' is not allowed for outputs"]});
 parse_name([$), T | S], Acc, 0, IsArg, State) ->
     Type =
         case T of

src/ejabberd_stun.erl

@@ -168,6 +168,8 @@ listen_opt_type(server_name) ->
 
 listen_options() ->
     [{shaper, none},
+     {use_turn, false},
+     {turn_ip, undefined},
      {auth_type, user},
      {auth_realm, undefined},
      {tls, false},

src/ejabberd_sup.erl

@@ -44,9 +44,9 @@ init([]) ->
 	   worker(ejabberd_ctl),
 	   worker(ejabberd_commands),
 	   worker(ejabberd_admin),
+	   supervisor(ejabberd_listener),
 	   worker(ejabberd_pkix),
 	   worker(ejabberd_acme),
-	   supervisor(ejabberd_listener),
 	   worker(ejabberd_s2s),
 	   simple_supervisor(ejabberd_s2s_in),
 	   simple_supervisor(ejabberd_s2s_out),

src/jd2ejd.erl

@@ -111,7 +111,6 @@ process_xdb(User, Server,
 xdb_data(_User, _Server, {xmlcdata, _CData}) -> ok;
 xdb_data(User, Server, #xmlel{attrs = Attrs} = El) ->
     From = jid:make(User, Server),
-    LUser = From#jid.luser,
     LServer = From#jid.lserver,
     case fxml:get_attr_s(<<"xmlns">>, Attrs) of
       ?NS_AUTH ->
@@ -142,7 +141,7 @@ xdb_data(User, Server, #xmlel{attrs = Attrs} = El) ->
 				(_) -> true
 			     end, Attrs),
 		catch mod_private:set_data(
-			LUser, LServer,
+			From,
 			[{XMLNS, El#xmlel{attrs = NewAttrs}}]);
 	    _ ->
 		?DEBUG("jd2ejd: Unknown namespace \"~s\"~n", [XMLNS])

src/misc.erl

@@ -28,7 +28,9 @@
 -module(misc).
 
 %% API
--export([tolower/1, term_to_base64/1, base64_to_term/1, ip_to_list/1,
+-export([add_delay_info/3, add_delay_info/4,
+	 unwrap_carbon/1, unwrap_mucsub_message/1, is_standalone_chat_state/1,
+	 tolower/1, term_to_base64/1, base64_to_term/1, ip_to_list/1,
 	 hex_to_bin/1, hex_to_base64/1, url_encode/1, expand_keyword/3,
 	 atom_to_binary/1, binary_to_atom/1, tuple_to_binary/1,
 	 l2i/1, i2l/1, i2l/2, expr_to_term/1, term_to_expr/1,
@@ -44,11 +46,81 @@
 	     {encode_base64, 1}]).
 
 -include("logger.hrl").
+-include("xmpp.hrl").
 -include_lib("kernel/include/file.hrl").
 
 %%%===================================================================
 %%% API
 %%%===================================================================
+-spec add_delay_info(stanza(), jid(), erlang:timestamp()) -> stanza().
+add_delay_info(Stz, From, Time) ->
+    add_delay_info(Stz, From, Time, <<"">>).
+
+-spec add_delay_info(stanza(), jid(), erlang:timestamp(), binary()) -> stanza().
+add_delay_info(Stz, From, Time, Desc) ->
+    NewDelay = #delay{stamp = Time, from = From, desc = Desc},
+    case xmpp:get_subtag(Stz, #delay{stamp = {0,0,0}}) of
+	#delay{from = OldFrom} when is_record(OldFrom, jid) ->
+	    case jid:tolower(From) == jid:tolower(OldFrom) of
+		true ->
+		    Stz;
+		false ->
+		    xmpp:append_subtags(Stz, [NewDelay])
+	    end;
+	_ ->
+	    xmpp:append_subtags(Stz, [NewDelay])
+    end.
+
+-spec unwrap_carbon(stanza()) -> xmpp_element().
+unwrap_carbon(#message{} = Msg) ->
+    try
+	case xmpp:get_subtag(Msg, #carbons_sent{forwarded = #forwarded{}}) of
+	    #carbons_sent{forwarded = #forwarded{sub_els = [El]}} ->
+		xmpp:decode(El, ?NS_CLIENT, [ignore_els]);
+	    _ ->
+		case xmpp:get_subtag(Msg, #carbons_received{
+					      forwarded = #forwarded{}}) of
+		    #carbons_received{forwarded = #forwarded{sub_els = [El]}} ->
+			xmpp:decode(El, ?NS_CLIENT, [ignore_els]);
+		    _ ->
+			Msg
+		end
+	end
+    catch _:{xmpp_codec, _} ->
+	    Msg
+    end;
+unwrap_carbon(Stanza) -> Stanza.
+
+-spec unwrap_mucsub_message(xmpp_element()) -> message() | false.
+unwrap_mucsub_message(#message{} = OuterMsg) ->
+    case xmpp:get_subtag(OuterMsg, #ps_event{}) of
+	#ps_event{
+	    items = #ps_items{
+		node = Node,
+		items = [
+		    #ps_item{
+			sub_els = [#message{} = InnerMsg]} | _]}}
+	    when Node == ?NS_MUCSUB_NODES_MESSAGES;
+		 Node == ?NS_MUCSUB_NODES_SUBJECT ->
+	    InnerMsg;
+	_ ->
+	    false
+    end;
+unwrap_mucsub_message(_Packet) ->
+    false.
+
+-spec is_standalone_chat_state(stanza()) -> boolean().
+is_standalone_chat_state(Stanza) ->
+    case unwrap_carbon(Stanza) of
+	#message{body = [], subject = [], sub_els = Els} ->
+	    IgnoreNS = [?NS_CHATSTATES, ?NS_DELAY, ?NS_EVENT],
+	    Stripped = [El || El <- Els,
+			      not lists:member(xmpp:get_ns(El), IgnoreNS)],
+	    Stripped == [];
+	_ ->
+	    false
+    end.
+
 -spec tolower(binary()) -> binary().
 tolower(B) ->
     iolist_to_binary(tolower_s(binary_to_list(B))).
@@ -204,7 +276,7 @@ compile_exprs(Mod, Exprs) ->
 
 -spec join_atoms([atom()], binary()) -> binary().
 join_atoms(Atoms, Sep) ->
-    str:join([io_lib:format("~p", [A]) || A <- Atoms], Sep).
+    str:join([io_lib:format("~p", [A]) || A <- lists:sort(Atoms)], Sep).
 
 %% @doc Checks if the file is readable and converts its name to binary.
 %%      Fails with `badarg` otherwise. The function is intended for usage

src/mod_admin_extra.erl

@@ -560,8 +560,10 @@ get_commands_spec() ->
 			desc = "Push template roster from file to a user",
 			longdesc = "The text file must contain an erlang term: a list "
 			    "of tuples with username, servername, group and nick. Example:\n"
-			    "[{\"user1\", \"localhost\", \"Workers\", \"User 1\"},\n"
-			    " {\"user2\", \"localhost\", \"Workers\", \"User 2\"}].",
+			    "[{<<\"user1\">>, <<\"localhost\">>, <<\"Workers\">>, <<\"User 1\">>},\n"
+			    " {<<\"user2\">>, <<\"localhost\">>, <<\"Workers\">>, <<\"User 2\">>}].\n"
+			    "When using UTF8 character encoding add /utf8 to certain string. Example:\n"
+			    "[{<<\"user2\">>, <<\"localhost\">>, <<\"Workers\"/utf8>>, <<\"User 2\"/utf8>>}].",
 			module = ?MODULE, function = push_roster,
 			args = [{file, binary}, {user, binary}, {host, binary}],
 			args_example = [<<"/home/ejabberd/roster.txt">>, <<"user1">>, <<"localhost">>],
@@ -1152,8 +1154,7 @@ set_vcard_content(User, Server, Data, SomeContent) ->
 	 end,
     %% Build new vcard
     SubEl = {xmlel, <<"vCard">>, [{<<"xmlns">>,<<"vcard-temp">>}], A4},
-    mod_vcard:set_vcard(User, jid:nameprep(Server), SubEl),
-    ok.
+    mod_vcard:set_vcard(User, jid:nameprep(Server), SubEl).
 
 take_vcard_tel(TelType, [{xmlel, <<"TEL">>, _, SubEls}=OldEl | OldEls], NewEls, Taken) ->
     {Taken2, NewEls2} = case lists:keymember(TelType, 2, SubEls) of
@@ -1389,9 +1390,8 @@ private_set(Username, Host, ElementString) ->
 
 private_set2(Username, Host, Xml) ->
     NS = fxml:get_tag_attr_s(<<"xmlns">>, Xml),
-    mod_private:set_data(jid:nodeprep(Username), jid:nameprep(Host),
-			 [{NS, Xml}]),
-    ok.
+    JID = jid:make(Username, Host),
+    mod_private:set_data(JID, [{NS, Xml}]).
 
 %%%
 %%% Shared Roster Groups

src/mod_avatar.erl

@@ -302,6 +302,10 @@ publish_avatar(#iq{from = JID} = IQ, Meta, MimeType, Data, ItemID) ->
 			       [jid:encode(JID), StanzaErr]),
 		    {stop, StanzaErr}
 	    end;
+	{error, #stanza_error{reason = 'not-acceptable'} = StanzaErr} ->
+	    ?WARNING_MSG("Failed to publish avatar data for ~s: ~p",
+			 [jid:encode(JID), StanzaErr]),
+	    {stop, StanzaErr};
 	{error, StanzaErr} ->
 	    ?ERROR_MSG("Failed to publish avatar data for ~s: ~p",
 		       [jid:encode(JID), StanzaErr]),

src/mod_block_strangers.erl

@@ -222,31 +222,11 @@ check_subscription(From, To) ->
     end.
 
 sets_bare_member({U, S, <<"">>} = LBJID, Set) ->
-    case ?SETS:next(sets_iterator_from(LBJID, Set)) of
+    case ?SETS:next(?SETS:iterator_from(LBJID, Set)) of
         {{U, S, _}, _} -> true;
         _ -> false
     end.
 
--ifdef(GB_SETS_ITERATOR_FROM).
-sets_iterator_from(Element, Set) ->
-    ?SETS:iterator_from(Element, Set).
--else.
-%% Copied from gb_sets.erl
-%% TODO: Remove after dropping R17 support
-sets_iterator_from(S, {_, T}) ->
-    iterator_from(S, T, []).
-
-iterator_from(S, {K, _, T}, As) when K < S ->
-    iterator_from(S, T, As);
-iterator_from(_, {_, nil, _} = T, As) ->
-    [T | As];
-iterator_from(S, {_, L, _} = T, As) ->
-    iterator_from(S, L, [T | As]);
-iterator_from(_, nil, As) ->
-    As.
--endif.
-
-
 depends(_Host, _Opts) ->
     [].
 

src/mod_carboncopy.erl

@@ -35,38 +35,25 @@
 -export([start/2, stop/1, reload/3]).
 
 -export([user_send_packet/1, user_receive_packet/1,
-	 iq_handler/1, remove_connection/4, disco_features/5,
-	 is_carbon_copy/1, mod_opt_type/1, depends/2, clean_cache/1,
+	 iq_handler/1, disco_features/5,
+	 is_carbon_copy/1, mod_opt_type/1, depends/2,
 	 mod_options/1]).
+%% For debugging purposes
+-export([list/2]).
 
 -include("logger.hrl").
 -include("xmpp.hrl").
--include("mod_carboncopy.hrl").
 
 -type direction() :: sent | received.
 
--callback init(binary(), gen_mod:opts()) -> any().
--callback enable(binary(), binary(), binary(), binary()) -> ok | {error, any()}.
--callback disable(binary(), binary(), binary()) -> ok | {error, any()}.
--callback list(binary(), binary()) -> [{binary(), binary(), node()}].
--callback use_cache(binary()) -> boolean().
--callback cache_nodes(binary()) -> [node()].
-
--optional_callbacks([use_cache/1, cache_nodes/1]).
-
 -spec is_carbon_copy(stanza()) -> boolean().
 is_carbon_copy(#message{meta = #{carbon_copy := true}}) ->
     true;
 is_carbon_copy(_) ->
     false.
 
-start(Host, Opts) ->
+start(Host, _Opts) ->
     ejabberd_hooks:add(disco_local_features, Host, ?MODULE, disco_features, 50),
-    Mod = gen_mod:ram_db_mod(Host, ?MODULE),
-    init_cache(Mod, Host, Opts),
-    Mod:init(Host, Opts),
-    clean_cache(),
-    ejabberd_hooks:add(unset_presence_hook,Host, ?MODULE, remove_connection, 10),
     %% why priority 89: to define clearly that we must run BEFORE mod_logdb hook (90)
     ejabberd_hooks:add(user_send_packet,Host, ?MODULE, user_send_packet, 89),
     ejabberd_hooks:add(user_receive_packet,Host, ?MODULE, user_receive_packet, 89),
@@ -77,23 +64,10 @@ stop(Host) ->
     ejabberd_hooks:delete(disco_local_features, Host, ?MODULE, disco_features, 50),
     %% why priority 89: to define clearly that we must run BEFORE mod_logdb hook (90)
     ejabberd_hooks:delete(user_send_packet,Host, ?MODULE, user_send_packet, 89),
-    ejabberd_hooks:delete(user_receive_packet,Host, ?MODULE, user_receive_packet, 89),
-    ejabberd_hooks:delete(unset_presence_hook,Host, ?MODULE, remove_connection, 10).
+    ejabberd_hooks:delete(user_receive_packet,Host, ?MODULE, user_receive_packet, 89).
 
-reload(Host, NewOpts, OldOpts) ->
-    NewMod = gen_mod:ram_db_mod(Host, NewOpts, ?MODULE),
-    OldMod = gen_mod:ram_db_mod(Host, OldOpts, ?MODULE),
-    if NewMod /= OldMod ->
-	    NewMod:init(Host, NewOpts);
-       true ->
-	    ok
-    end,
-    case use_cache(NewMod, Host) of
-	true ->
-	    ets_cache:new(?CARBONCOPY_CACHE, cache_opts(NewOpts));
-	false ->
-	    ok
-    end.
+reload(_Host, _NewOpts, _OldOpts) ->
+    ok.
 
 -spec disco_features({error, stanza_error()} | {result, [binary()]} | empty,
 		     jid(), jid(), binary(), binary()) ->
@@ -113,19 +87,13 @@ iq_handler(#iq{type = set, lang = Lang, from = From,
 					  is_record(El, carbons_disable) ->
     {U, S, R} = jid:tolower(From),
     Result = case El of
-		 #carbons_enable{} ->
-		     ?DEBUG("Carbons enabled for user ~s@~s/~s", [U,S,R]),
-		     enable(S, U, R, ?NS_CARBONS_2);
-		 #carbons_disable{} ->
-		     ?DEBUG("Carbons disabled for user ~s@~s/~s", [U,S,R]),
-		     disable(S, U, R)
+		 #carbons_enable{} -> enable(S, U, R, ?NS_CARBONS_2);
+		 #carbons_disable{} -> disable(S, U, R)
 	     end,
     case Result of
 	ok ->
-	    ?DEBUG("carbons IQ result: ok", []),
 	    xmpp:make_iq_result(IQ);
-	{error,_Error} ->
-	    ?ERROR_MSG("Error enabling / disabling carbons: ~p", [Result]),
+	{error, _} ->
 	    Txt = <<"Database failure">>,
 	    xmpp:make_error(IQ, xmpp:err_internal_server_error(Txt, Lang))
     end;
@@ -180,12 +148,6 @@ check_and_forward(JID, To, Packet, Direction)->
 	    Packet
     end.
 
--spec remove_connection(binary(), binary(), binary(), binary()) -> ok.
-remove_connection(User, Server, Resource, _Status)->
-    disable(Server, User, Resource),
-    ok.
-
-
 %%% Internal
 %% Direction = received | sent <received xmlns='urn:xmpp:carbons:1'/>
 -spec send_copies(jid(), jid(), message(), direction()) -> ok.
@@ -248,22 +210,26 @@ build_forward_packet(JID, #message{type = T} = Msg, Sender, Dest, Direction) ->
 
 -spec enable(binary(), binary(), binary(), binary()) -> ok | {error, any()}.
 enable(Host, U, R, CC)->
-    ?DEBUG("enabling for ~p", [U]),
-    Mod = gen_mod:ram_db_mod(Host, ?MODULE),
-    case Mod:enable(U, Host, R, CC) of
-	ok ->
-	    delete_cache(Mod, U, Host);
-	{error, _} = Err ->
+    ?DEBUG("Enabling carbons for ~s@~s/~s", [U, Host, R]),
+    case ejabberd_sm:set_user_info(U, Host, R, carboncopy, CC) of
+	ok -> ok;
+	{error, Reason} = Err ->
+	    ?ERROR_MSG("Failed to disable carbons for ~s@~s/~s: ~p",
+		       [U, Host, R, Reason]),
 	    Err
     end.
 
 -spec disable(binary(), binary(), binary()) -> ok | {error, any()}.
 disable(Host, U, R)->
-    ?DEBUG("disabling for ~p", [U]),
-    Mod = gen_mod:ram_db_mod(Host, ?MODULE),
-    Res = Mod:disable(U, Host, R),
-    delete_cache(Mod, U, Host),
-    Res.
+    ?DEBUG("Disabling carbons for ~s@~s/~s", [U, Host, R]),
+    case ejabberd_sm:del_user_info(U, Host, R, carboncopy) of
+	ok -> ok;
+	{error, notfound} -> ok;
+	{error, Reason} = Err ->
+	    ?ERROR_MSG("Failed to disable carbons for ~s@~s/~s: ~p",
+		       [U, Host, R, Reason]),
+	    Err
+    end.
 
 -spec complete_packet(jid(), message(), direction()) -> message().
 complete_packet(From, #message{from = undefined} = Msg, sent) ->
@@ -291,99 +257,30 @@ is_received_muc_pm(_To, Packet, received) ->
 
 -spec list(binary(), binary()) -> [{Resource :: binary(), Namespace :: binary()}].
 list(User, Server) ->
-    Mod = gen_mod:ram_db_mod(Server, ?MODULE),
-    case use_cache(Mod, Server) of
-	true ->
-	    case ets_cache:lookup(
-		   ?CARBONCOPY_CACHE, {User, Server},
-		   fun() ->
-			   case Mod:list(User, Server) of
-			       {ok, L} when L /= [] -> {ok, L};
-			       _ -> error
-			   end
-		   end) of
-		{ok, L} -> [{Resource, NS} || {Resource, NS, _} <- L];
-		error -> []
-	    end;
-	false ->
-	    case Mod:list(User, Server) of
-		{ok, L} -> [{Resource, NS} || {Resource, NS, _} <- L];
-		error -> []
-	    end
-    end.
-
--spec init_cache(module(), binary(), gen_mod:opts()) -> ok.
-init_cache(Mod, Host, Opts) ->
-    case use_cache(Mod, Host) of
-	true ->
-	    ets_cache:new(?CARBONCOPY_CACHE, cache_opts(Opts));
-	false ->
-	    ets_cache:delete(?CARBONCOPY_CACHE)
-    end.
-
--spec cache_opts(gen_mod:opts()) -> [proplists:property()].
-cache_opts(Opts) ->
-    MaxSize = gen_mod:get_opt(cache_size, Opts),
-    CacheMissed = gen_mod:get_opt(cache_missed, Opts),
-    LifeTime = case gen_mod:get_opt(cache_life_time, Opts) of
-		   infinity -> infinity;
-		   I -> timer:seconds(I)
-	       end,
-    [{max_size, MaxSize}, {cache_missed, CacheMissed}, {life_time, LifeTime}].
-
--spec use_cache(module(), binary()) -> boolean().
-use_cache(Mod, Host) ->
-    case erlang:function_exported(Mod, use_cache, 1) of
-	true -> Mod:use_cache(Host);
-	false -> gen_mod:get_module_opt(Host, ?MODULE, use_cache)
-    end.
-
--spec cache_nodes(module(), binary()) -> [node()].
-cache_nodes(Mod, Host) ->
-    case erlang:function_exported(Mod, cache_nodes, 1) of
-	true -> Mod:cache_nodes(Host);
-	false -> ejabberd_cluster:get_nodes()
-    end.
-
--spec clean_cache(node()) -> non_neg_integer().
-clean_cache(Node) ->
-    ets_cache:filter(
-      ?CARBONCOPY_CACHE,
-      fun(_, error) ->
-	      false;
-	 (_, {ok, L}) ->
-	      not lists:any(fun({_, _, N}) -> N == Node end, L)
-      end).
-
--spec clean_cache() -> ok.
-clean_cache() ->
-    ejabberd_cluster:eval_everywhere(?MODULE, clean_cache, [node()]).
-
--spec delete_cache(module(), binary(), binary()) -> ok.
-delete_cache(Mod, User, Server) ->
-    case use_cache(Mod, Server) of
-	true ->
-	    ets_cache:delete(?CARBONCOPY_CACHE, {User, Server},
-			     cache_nodes(Mod, Server));
-	false ->
-	    ok
-    end.
+    lists:filtermap(
+      fun({Resource, Info}) ->
+	      case lists:keyfind(carboncopy, 1, Info) of
+		  {_, NS} -> {true, {Resource, NS}};
+		  false -> false
+	      end
+      end, ejabberd_sm:get_user_info(User, Server)).
 
 depends(_Host, _Opts) ->
     [].
 
-mod_opt_type(ram_db_type) -> fun(T) -> ejabberd_config:v_db(?MODULE, T) end;
-mod_opt_type(O) when O == use_cache; O == cache_missed ->
-    fun(B) when is_boolean(B) -> B end;
-mod_opt_type(O) when O == cache_size; O == cache_life_time ->
-    fun(I) when is_integer(I), I>0 -> I;
-       (unlimited) -> infinity;
-       (infinity) -> infinity
+mod_opt_type(O) when O == cache_size; O == cache_life_time;
+		     O == use_cache; O == cache_missed;
+		     O == ram_db_type ->
+    fun(deprecated) -> deprecated;
+       (_) ->
+	    ?WARNING_MSG("Option ~s of ~s has no effect anymore "
+			 "and will be ingored", [O, ?MODULE]),
+	    deprecated
     end.
 
-mod_options(Host) ->
-    [{ram_db_type, ejabberd_config:default_ram_db(Host, ?MODULE)},
-     {use_cache, ejabberd_config:use_cache(Host)},
-     {cache_size, ejabberd_config:cache_size(Host)},
-     {cache_missed, ejabberd_config:cache_missed(Host)},
-     {cache_life_time, ejabberd_config:cache_life_time(Host)}].
+mod_options(_) ->
+    [{ram_db_type, deprecated},
+     {use_cache, deprecated},
+     {cache_size, deprecated},
+     {cache_missed, deprecated},
+     {cache_life_time, deprecated}].

src/mod_carboncopy_mnesia.erl

@@ -1,79 +0,0 @@
-%%%-------------------------------------------------------------------
-%%% File    : mod_carboncopy_mnesia.erl
-%%% Author  : Evgeny Khramtsov <ekhramtsov@process-one.net>
-%%% Created : 15 Apr 2016 by Evgeny Khramtsov <ekhramtsov@process-one.net>
-%%%
-%%%
-%%% ejabberd, Copyright (C) 2002-2018   ProcessOne
-%%%
-%%% This program is free software; you can redistribute it and/or
-%%% modify it under the terms of the GNU General Public License as
-%%% published by the Free Software Foundation; either version 2 of the
-%%% License, or (at your option) any later version.
-%%%
-%%% This program is distributed in the hope that it will be useful,
-%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-%%% General Public License for more details.
-%%%
-%%% You should have received a copy of the GNU General Public License along
-%%% with this program; if not, write to the Free Software Foundation, Inc.,
-%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-%%%
-%%%----------------------------------------------------------------------
-
--module(mod_carboncopy_mnesia).
-
--behaviour(mod_carboncopy).
-
-%% API
--export([init/2, enable/4, disable/3, list/2, use_cache/1]).
-
--include("mod_carboncopy.hrl").
-
-%%%===================================================================
-%%% API
-%%%===================================================================
-init(_Host, _Opts) ->
-    Fields = record_info(fields, carboncopy),
-    try mnesia:table_info(carboncopy, attributes) of
-	Fields ->
-	    ok;
-	_ ->
-	    %% recreate..
-	    mnesia:delete_table(carboncopy)
-    catch _:_Error ->
-	    %% probably table don't exist
-	    ok
-    end,
-    ejabberd_mnesia:create(?MODULE, carboncopy,
-			[{ram_copies, [node()]}, 
-			 {attributes, record_info(fields, carboncopy)}, 
-			 {type, bag}]).
-
-enable(LUser, LServer, LResource, NS) ->
-    mnesia:dirty_write(
-      #carboncopy{us = {LUser, LServer},
-		  resource = LResource,
-		  version = NS}).
-
-disable(LUser, LServer, LResource) ->
-    ToDelete = mnesia:dirty_match_object(
-		 #carboncopy{us = {LUser, LServer},
-			     resource = LResource,
-			     _ = '_'}),
-    lists:foreach(fun mnesia:dirty_delete_object/1, ToDelete).
-
-list(LUser, LServer) ->
-    {ok, mnesia:dirty_select(
-	   carboncopy,
-	   [{#carboncopy{us = {LUser, LServer}, resource = '$2',
-			 version = '$3', node = '$4'},
-	     [], [{{'$2','$3','$4'}}]}])}.
-
-use_cache(_LServer) ->
-    false.
-
-%%%===================================================================
-%%% Internal functions
-%%%===================================================================

src/mod_carboncopy_redis.erl

@@ -1,176 +0,0 @@
-%%%-------------------------------------------------------------------
-%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
-%%% Created : 30 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
-%%%
-%%%
-%%% ejabberd, Copyright (C) 2002-2018   ProcessOne
-%%%
-%%% This program is free software; you can redistribute it and/or
-%%% modify it under the terms of the GNU General Public License as
-%%% published by the Free Software Foundation; either version 2 of the
-%%% License, or (at your option) any later version.
-%%%
-%%% This program is distributed in the hope that it will be useful,
-%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-%%% General Public License for more details.
-%%%
-%%% You should have received a copy of the GNU General Public License along
-%%% with this program; if not, write to the Free Software Foundation, Inc.,
-%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-%%%
-%%%-------------------------------------------------------------------
--module(mod_carboncopy_redis).
--behaviour(mod_carboncopy).
--behaviour(gen_server).
-
-%% API
--export([init/2, enable/4, disable/3, list/2, cache_nodes/1]).
-%% gen_server callbacks
--export([init/1, handle_cast/2, handle_call/3, handle_info/2,
-	 terminate/2, code_change/3, start_link/0]).
-
--include("logger.hrl").
--include("mod_carboncopy.hrl").
-
--define(CARBONCOPY_KEY, <<"ejabberd:carboncopy">>).
-
--record(state, {}).
-
-%%%===================================================================
-%%% API
-%%%===================================================================
-init(_Host, _Opts) ->
-    Spec = {?MODULE, {?MODULE, start_link, []},
-	    transient, 5000, worker, [?MODULE]},
-    case supervisor:start_child(ejabberd_backend_sup, Spec) of
-	{ok, _Pid} -> ok;
-	Err -> Err
-    end.
-
--spec start_link() -> {ok, pid()} | {error, any()}.
-start_link() ->
-    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
-
-cache_nodes(_LServer) ->
-    [node()].
-
-enable(LUser, LServer, LResource, NS) ->
-    USKey = us_key(LUser, LServer),
-    NodeKey = node_key(),
-    JID = jid:encode({LUser, LServer, LResource}),
-    Data = term_to_binary({NS, node()}),
-    case ejabberd_redis:multi(
-	   fun() ->
-		   ejabberd_redis:hset(USKey, LResource, Data),
-		   ejabberd_redis:sadd(NodeKey, [JID]),
-		   ejabberd_redis:publish(
-		     ?CARBONCOPY_KEY,
-		     term_to_binary({delete, {LUser, LServer}}))
-	   end) of
-	{ok, _} ->
-	    ok;
-	{error, _} ->
-	    {error, db_failure}
-    end.
-
-disable(LUser, LServer, LResource) ->
-    USKey = us_key(LUser, LServer),
-    NodeKey = node_key(),
-    JID = jid:encode({LUser, LServer, LResource}),
-    case ejabberd_redis:multi(
-	   fun() ->
-		   ejabberd_redis:hdel(USKey, [LResource]),
-		   ejabberd_redis:srem(NodeKey, [JID]),
-		   ejabberd_redis:publish(
-		     ?CARBONCOPY_KEY,
-		     term_to_binary({delete, {LUser, LServer}}))
-	   end) of
-	{ok, _} ->
-	    ok;
-	{error, _} ->
-	    {error, db_failure}
-    end.
-
-list(LUser, LServer) ->
-    USKey = us_key(LUser, LServer),
-    case ejabberd_redis:hgetall(USKey) of
-	{ok, Pairs} ->
-	    {ok, lists:flatmap(
-		   fun({Resource, Data}) ->
-			   try
-			       {NS, Node} = binary_to_term(Data),
-			       [{Resource, NS, Node}]
-			   catch _:_ ->
-				   ?ERROR_MSG("invalid term stored in Redis "
-					      "(key = ~s): ~p",
-					      [USKey, Data]),
-				   []
-			   end
-		   end, Pairs)};
-	{error, _} ->
-	    {error, db_failure}
-    end.
-
-%%%===================================================================
-%%% gen_server callbacks
-%%%===================================================================
-init([]) ->
-    ejabberd_redis:subscribe([?CARBONCOPY_KEY]),
-    clean_table(),
-    {ok, #state{}}.
-
-handle_call(_Request, _From, State) ->
-    Reply = ok,
-    {reply, Reply, State}.
-
-handle_cast(_Msg, State) ->
-    {noreply, State}.
-
-handle_info({redis_message, ?CARBONCOPY_KEY, Data}, State) ->
-    case binary_to_term(Data) of
-	{delete, Key} ->
-	    ets_cache:delete(?CARBONCOPY_CACHE, Key);
-	Msg ->
-	    ?WARNING_MSG("unexpected redis message: ~p", [Msg])
-    end,
-    {noreply, State};
-handle_info(Info, State) ->
-    ?ERROR_MSG("unexpected info: ~p", [Info]),
-    {noreply, State}.
-
-terminate(_Reason, _State) ->
-    ok.
-
-code_change(_OldVsn, State, _Extra) ->
-    {ok, State}.
-
-%%%===================================================================
-%%% Internal functions
-%%%===================================================================
-clean_table() ->
-    ?DEBUG("Cleaning Redis 'carboncopy' table...", []),
-    NodeKey = node_key(),
-    case ejabberd_redis:smembers(NodeKey) of
-	{ok, JIDs} ->
-	    ejabberd_redis:multi(
-	      fun() ->
-		      lists:foreach(
-			fun(JID) ->
-				{U, S, R} = jid:split(jid:decode(JID)),
-				USKey = us_key(U, S),
-				ejabberd_redis:hdel(USKey, [R])
-			end, JIDs)
-	      end);
-	{error, _} ->
-	    ok
-    end,
-    ejabberd_redis:del([NodeKey]),
-    ok.
-
-us_key(LUser, LServer) ->
-    <<"ejabberd:carboncopy:users:", LUser/binary, $@, LServer/binary>>.
-
-node_key() ->
-    Node = erlang:atom_to_binary(node(), latin1),
-    <<"ejabberd:carboncopy:nodes:", Node/binary>>.

src/mod_carboncopy_riak.erl

@@ -1,82 +0,0 @@
-%%%-------------------------------------------------------------------
-%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
-%%% Created : 15 Apr 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
-%%%
-%%%
-%%% ejabberd, Copyright (C) 2002-2018   ProcessOne
-%%%
-%%% This program is free software; you can redistribute it and/or
-%%% modify it under the terms of the GNU General Public License as
-%%% published by the Free Software Foundation; either version 2 of the
-%%% License, or (at your option) any later version.
-%%%
-%%% This program is distributed in the hope that it will be useful,
-%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-%%% General Public License for more details.
-%%%
-%%% You should have received a copy of the GNU General Public License along
-%%% with this program; if not, write to the Free Software Foundation, Inc.,
-%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-%%%
-%%%-------------------------------------------------------------------
--module(mod_carboncopy_riak).
--behaviour(mod_carboncopy).
-
-%% API
--export([init/2, enable/4, disable/3, list/2]).
-
--include("logger.hrl").
--include("mod_carboncopy.hrl").
-
-%%%===================================================================
-%%% API
-%%%===================================================================
-init(_Host, _Opts) ->
-    clean_table().
-
-enable(LUser, LServer, LResource, NS) ->
-    ejabberd_riak:put(#carboncopy{us = {LUser, LServer},
-				  resource = LResource,
-				  version = NS},
-		      carboncopy_schema(),
-		      [{i, {LUser, LServer, LResource}},
-		       {'2i', [{<<"us">>, {LUser, LServer}}]}]).
-
-disable(LUser, LServer, LResource) ->
-    ejabberd_riak:delete(carboncopy, {LUser, LServer, LResource}).
-
-list(LUser, LServer) ->
-    case ejabberd_riak:get_by_index(
-	   carboncopy, carboncopy_schema(),
-	   <<"us">>, {LUser, LServer}) of
-	{ok, Rs} ->
-	    {ok, [{Resource, NS, Node}
-		  || #carboncopy{resource = Resource,
-				 version = NS,
-				 node = Node} <- Rs]};
-	{error, _} = Err ->
-	    Err
-    end.
-
-%%%===================================================================
-%%% Internal functions
-%%%===================================================================
-carboncopy_schema() ->
-    {record_info(fields, carboncopy), #carboncopy{}}.
-
-clean_table() ->
-    ?DEBUG("Cleaning Riak 'carboncopy' table...", []),
-    case ejabberd_riak:get(carboncopy, carboncopy_schema()) of
-	{ok, Rs} ->
-	    lists:foreach(
-	      fun(#carboncopy{us = {U, S}, resource = R, node = Node})
-		    when Node == node() ->
-		      ejabberd_riak:delete(carboncopy, {U, S, R});
-		 (_) ->
-		      ok
-	      end, Rs);
-	{error, Reason} = Err ->
-	    ?ERROR_MSG("Failed to clean Riak 'carboncopy' table: ~p", [Reason]),
-	    Err
-    end.

src/mod_carboncopy_sql.erl

@@ -1,91 +0,0 @@
-%%%-------------------------------------------------------------------
-%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
-%%% Created : 29 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
-%%%
-%%%
-%%% ejabberd, Copyright (C) 2002-2018   ProcessOne
-%%%
-%%% This program is free software; you can redistribute it and/or
-%%% modify it under the terms of the GNU General Public License as
-%%% published by the Free Software Foundation; either version 2 of the
-%%% License, or (at your option) any later version.
-%%%
-%%% This program is distributed in the hope that it will be useful,
-%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
-%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-%%% General Public License for more details.
-%%%
-%%% You should have received a copy of the GNU General Public License along
-%%% with this program; if not, write to the Free Software Foundation, Inc.,
-%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-%%%
-%%%-------------------------------------------------------------------
--module(mod_carboncopy_sql).
--behaviour(mod_carboncopy).
-
--compile([{parse_transform, ejabberd_sql_pt}]).
-
-%% API
--export([init/2, enable/4, disable/3, list/2]).
-
--include("logger.hrl").
--include("ejabberd_sql_pt.hrl").
-
-%%%===================================================================
-%%% API
-%%%===================================================================
-init(Host, _Opts) ->
-    clean_table(Host).
-
-enable(LUser, LServer, LResource, NS) ->
-    NodeS = erlang:atom_to_binary(node(), latin1),
-    case ?SQL_UPSERT(LServer, "carboncopy",
-		     ["!username=%(LUser)s",
-                      "!server_host=%(LServer)s",
-		      "!resource=%(LResource)s",
-		      "namespace=%(NS)s",
-		      "node=%(NodeS)s"]) of
-	ok ->
-	    ok;
-	_Err ->
-	    {error, db_failure}
-    end.
-
-disable(LUser, LServer, LResource) ->
-    case ejabberd_sql:sql_query(
-	   LServer,
-	   ?SQL("delete from carboncopy where username=%(LUser)s "
-		"and %(LServer)H and resource=%(LResource)s")) of
-	{updated, _} ->
-	    ok;
-	_Err ->
-	    {error, db_failure}
-    end.
-
-list(LUser, LServer) ->
-    case ejabberd_sql:sql_query(
-	   LServer,
-	   ?SQL("select @(resource)s, @(namespace)s, @(node)s from carboncopy "
-		"where username=%(LUser)s and %(LServer)H")) of
-	{selected, Rows} ->
-	    {ok, [{Resource, NS, binary_to_atom(Node, latin1)}
-		  || {Resource, NS, Node} <- Rows]};
-	_Err ->
-	    {error, db_failure}
-    end.
-
-%%%===================================================================
-%%% Internal functions
-%%%===================================================================
-clean_table(LServer) ->
-    NodeS = erlang:atom_to_binary(node(), latin1),
-    ?DEBUG("Cleaning SQL 'carboncopy' table...", []),
-    case ejabberd_sql:sql_query(
-	   LServer,
-	   ?SQL("delete from carboncopy where node=%(NodeS)s")) of
-	{updated, _} ->
-	    ok;
-	Err ->
-	    ?ERROR_MSG("failed to clean 'carboncopy' table: ~p", [Err]),
-	    {error, db_failure}
-    end.

src/mod_client_state.erl

@@ -237,7 +237,7 @@ filter_chat_states({#message{meta = #{csi_resend := true}}, _} = Acc) ->
     Acc;
 filter_chat_states({#message{from = From, to = To} = Msg,
 		    #{csi_state := inactive} = C2SState} = Acc) ->
-    case xmpp_util:is_standalone_chat_state(Msg) of
+    case misc:is_standalone_chat_state(Msg) of
 	true ->
 	    case {From, To} of
 		{#jid{luser = U, lserver = S}, #jid{luser = U, lserver = S}} ->
@@ -352,7 +352,7 @@ flush_stanzas(#{lserver := LServer} = C2SState, Elems) ->
 
 -spec add_delay_info(stanza(), binary(), csi_timestamp()) -> stanza().
 add_delay_info(Stanza, LServer, {_Seq, TimeStamp}) ->
-    Stanza1 = xmpp_util:add_delay_info(
+    Stanza1 = misc:add_delay_info(
 		Stanza, jid:make(LServer), TimeStamp,
 		<<"Client Inactive">>),
     xmpp:put_meta(Stanza1, csi_resend, true).

src/mod_fail2ban.erl

@@ -36,7 +36,11 @@
 	 handle_info/2, terminate/2, code_change/3,
 	 mod_opt_type/1, mod_options/1, depends/2]).
 
+%% ejabberd command.
+-export([get_commands_spec/0, unban/1]).
+
 -include_lib("stdlib/include/ms_transform.hrl").
+-include("ejabberd_commands.hrl").
 -include("logger.hrl").
 -include("xmpp.hrl").
 
@@ -101,9 +105,16 @@ c2s_stream_started(#{ip := {Addr, _}} = State, _) ->
 start(Host, Opts) ->
     catch ets:new(failed_auth, [named_table, public,
 				{heir, erlang:group_leader(), none}]),
+    ejabberd_commands:register_commands(get_commands_spec()),
     gen_mod:start_child(?MODULE, Host, Opts).
 
 stop(Host) ->
+    case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of
+        false ->
+            ejabberd_commands:unregister_commands(get_commands_spec());
+        true ->
+            ok
+    end,
     gen_mod:stop_child(?MODULE, Host).
 
 reload(_Host, _NewOpts, _OldOpts) ->
@@ -155,6 +166,46 @@ terminate(_Reason, #state{host = Host}) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
+%%--------------------------------------------------------------------
+%% ejabberd command callback.
+%%--------------------------------------------------------------------
+-spec get_commands_spec() -> [ejabberd_commands()].
+get_commands_spec() ->
+    [#ejabberd_commands{name = unban_ip, tags = [accounts],
+			desc = "Remove banned IP addresses from the fail2ban table",
+			longdesc = "Accepts an IP address with a network mask. "
+			    "Returns the number of unbanned addresses, or a negative integer if there were any error.",
+			module = ?MODULE, function = unban,
+			args = [{address, binary}],
+			args_example = [<<"::FFFF:127.0.0.1/128">>],
+			args_desc = ["IP address, optionally with network mask."],
+			result_example = 3,
+			result_desc = "Amount of unbanned entries, or negative in case of error.",
+			result = {unbanned, integer}}].
+
+-spec unban(string()) -> integer().
+unban(S) ->
+    case acl:parse_ip_netmask(S) of
+	{ok, Net, Mask} ->
+	    unban(Net, Mask);
+	error ->
+	    ?WARNING_MSG("Invalid network address when trying to unban: ~p", [S]),
+	    -1
+    end.
+
+unban(Net, Mask) ->
+    ets:foldl(
+	fun({Addr, _, _, _}, Acc)  ->
+	    case acl:ip_matches_mask(Addr, Net, Mask) of
+		true ->
+		    ets:delete(failed_auth, Addr),
+		    Acc+1;
+		false -> Acc
+	    end
+	end,
+	0,
+	failed_auth).
+
 %%%===================================================================
 %%% Internal functions
 %%%===================================================================

src/mod_http_api.erl

@@ -274,20 +274,8 @@ handle(Call, Auth, Args, Version) when is_atom(Call), is_list(Args) ->
     case ejabberd_commands:get_command_format(Call, Auth, Version) of
         {ArgsSpec, _} when is_list(ArgsSpec) ->
             Args2 = [{misc:binary_to_atom(Key), Value} || {Key, Value} <- Args],
-            Spec = lists:foldr(
-                    fun ({Key, binary}, Acc) ->
-                            [{Key, <<>>}|Acc];
-                        ({Key, string}, Acc) ->
-			    [{Key, ""}|Acc];
-                        ({Key, integer}, Acc) ->
-                            [{Key, 0}|Acc];
-                        ({Key, {list, _}}, Acc) ->
-                            [{Key, []}|Acc];
-                        ({Key, atom}, Acc) ->
-                            [{Key, undefined}|Acc]
-                    end, [], ArgsSpec),
 	    try
-          handle2(Call, Auth, match(Args2, Spec), Version)
+          handle2(Call, Auth, Args2, Version)
 	    catch throw:not_found ->
 		    {404, <<"not_found">>};
 		  throw:{not_found, Why} when is_atom(Why) ->
@@ -301,9 +289,9 @@ handle(Call, Auth, Args, Version) when is_atom(Call), is_list(Args) ->
 		  throw:{not_allowed, Msg} ->
 		    {401, iolist_to_binary(Msg)};
                   throw:{error, account_unprivileged} ->
-        {403, 31, <<"Command need to be run with admin privilege.">>};
-      throw:{error, access_rules_unauthorized} ->
-        {403, 32, <<"AccessRules: Account does not have the right to perform the operation.">>};
+		      {403, 31, <<"Command need to be run with admin privilege.">>};
+		throw:{error, access_rules_unauthorized} ->
+		    {403, 32, <<"AccessRules: Account does not have the right to perform the operation.">>};
 		  throw:{invalid_parameter, Msg} ->
 		    {400, iolist_to_binary(Msg)};
 		  throw:{error, Why} when is_atom(Why) ->
@@ -337,15 +325,20 @@ handle2(Call, Auth, Args, Version) when is_atom(Call), is_list(Args) ->
 	    format_command_result(Call, Auth, Res, Version)
     end.
 
-get_elem_delete(A, L) ->
+get_elem_delete(A, L, F) ->
     case proplists:get_all_values(A, L) of
       [Value] -> {Value, proplists:delete(A, L)};
       [_, _ | _] ->
 	  %% Crash reporting the error
 	  exit({duplicated_attribute, A, L});
       [] ->
-	  %% Report the error and then force a crash
-	  exit({attribute_not_found, A, L})
+	  case F of
+	      {list, _} ->
+		  {[], L};
+	      _ ->
+		  %% Report the error and then force a crash
+		  exit({attribute_not_found, A, L})
+	  end
     end.
 
 format_args(Args, ArgsFormat) ->
@@ -354,7 +347,7 @@ format_args(Args, ArgsFormat) ->
 					  {Args1, Res}) ->
 					     {ArgValue, Args2} =
 						 get_elem_delete(ArgName,
-								 Args1),
+								 Args1, ArgFormat),
 					     Formatted = format_arg(ArgValue,
 								    ArgFormat),
 					     {Args2, Res ++ [Formatted]}
@@ -431,9 +424,6 @@ process_unicode_codepoints(Str) ->
 %% internal helpers
 %% ----------------
 
-match(Args, Spec) ->
-    [{Key, proplists:get_value(Key, Args, Default)} || {Key, Default} <- Spec].
-
 format_command_result(Cmd, Auth, Result, Version) ->
     {_, ResultFormat} = ejabberd_commands:get_command_format(Cmd, Auth, Version),
     case {ResultFormat, Result} of
@@ -486,6 +476,9 @@ format_result(Code, {Name, restuple}) ->
 format_result(Els, {Name, {list, {_, {tuple, [{_, atom}, _]}} = Fmt}}) ->
     {misc:atom_to_binary(Name), {[format_result(El, Fmt) || El <- Els]}};
 
+format_result(Els, {Name, {list, {_, {tuple, [{name, string}, {value, _}]}} = Fmt}}) ->
+    {misc:atom_to_binary(Name), {[format_result(El, Fmt) || El <- Els]}};
+
 format_result(Els, {Name, {list, Def}}) ->
     {misc:atom_to_binary(Name), [element(2, format_result(El, Def)) || El <- Els]};
 
@@ -494,6 +487,11 @@ format_result(Tuple, {_Name, {tuple, [{_, atom}, ValFmt]}}) ->
     {_, Val2} = format_result(Val, ValFmt),
     {misc:atom_to_binary(Name2), Val2};
 
+format_result(Tuple, {_Name, {tuple, [{name, string}, {value, _} = ValFmt]}}) ->
+    {Name2, Val} = Tuple,
+    {_, Val2} = format_result(Val, ValFmt),
+    {iolist_to_binary(Name2), Val2};
+
 format_result(Tuple, {Name, {tuple, Def}}) ->
     Els = lists:zip(tuple_to_list(Tuple), Def),
     {misc:atom_to_binary(Name), {[format_result(El, ElDef) || {El, ElDef} <- Els]}};
@@ -504,6 +502,8 @@ format_result(404, {_Name, _}) ->
 
 format_error_result(conflict, Code, Msg) ->
     {409, Code, iolist_to_binary(Msg)};
+format_error_result(not_exists, Code, Msg) ->
+    {404, Code, iolist_to_binary(Msg)};
 format_error_result(_ErrorAtom, Code, Msg) ->
     {500, Code, iolist_to_binary(Msg)}.
 

src/mod_mam.erl

@@ -648,8 +648,20 @@ should_archive(#message{body = Body, subject = Subject,
 		none when Type == headline ->
 		    false;
 		none ->
-		    xmpp:get_text(Body) /= <<>> orelse
-			xmpp:get_text(Subject) /= <<>>
+		    case xmpp:get_text(Body) /= <<>> orelse
+			 xmpp:get_text(Subject) /= <<>> of
+			true ->
+			    true;
+			_ ->
+			    case misc:unwrap_mucsub_message(Pkt) of
+				#message{type = groupchat} = Msg ->
+				    should_archive(Msg#message{type = chat}, LServer);
+				#message{} = Msg ->
+				    should_archive(Msg, LServer);
+				_ ->
+				    false
+			    end
+		    end
 	    end
     end;
 should_archive(_, _LServer) ->
@@ -1160,7 +1172,7 @@ mod_opt_type(O) when O == cache_life_time; O == cache_size ->
     fun (I) when is_integer(I), I > 0 -> I;
 	(infinity) -> infinity
     end;
-mod_opt_type(O) when O == use_cache; O == cache_missed ->
+mod_opt_type(O) when O == use_cache; O == cache_missed; O == compress_xml ->
     fun (B) when is_boolean(B) -> B end;
 mod_opt_type(db_type) -> fun(T) -> ejabberd_config:v_db(?MODULE, T) end;
 mod_opt_type(default) ->
@@ -1175,6 +1187,7 @@ mod_options(Host) ->
     [{assume_mam_usage, false},
      {default, never},
      {request_activates_archiving, false},
+     {compress_xml, false},
      {db_type, ejabberd_config:default_db(Host, ?MODULE)},
      {use_cache, ejabberd_config:use_cache(Host)},
      {cache_size, ejabberd_config:cache_size(Host)},

src/mod_mam_sql.erl

@@ -102,9 +102,18 @@ store(Pkt, LServer, {LUser, LHost}, Type, Peer, Nick, _Dir, TS) ->
 		   jid:remove_resource(Peer))),
     LPeer = jid:encode(
 	      jid:tolower(Peer)),
-    XML = fxml:element_to_binary(Pkt),
     Body = fxml:get_subtag_cdata(Pkt, <<"body">>),
     SType = misc:atom_to_binary(Type),
+    XML = case gen_mod:get_module_opt(LServer, mod_mam, compress_xml) of
+	      true ->
+		  J1 = case Type of
+			      chat -> jid:encode({LUser, LHost, <<>>});
+			      groupchat -> SUser
+			  end,
+		  xml_compress:encode(Pkt, J1, LPeer);
+	      _ ->
+		  fxml:element_to_binary(Pkt)
+	  end,
     case ejabberd_sql:sql_query(
            LServer,
            ?SQL_INSERT(
@@ -192,8 +201,8 @@ select(LServer, JidRequestor, #jid{luser = LUser} = JidArchive,
 	    {lists:flatmap(
 	       fun([TS, XML, PeerBin, Kind, Nick]) ->
 		       case make_archive_el(
-			      TS, XML, PeerBin, Kind, Nick,
-			      MsgType, JidRequestor, JidArchive) of
+			   jid:encode(JidArchive), TS, XML, PeerBin, Kind, Nick,
+			   MsgType, JidRequestor, JidArchive) of
 			   {ok, El} ->
 			       [{TS, binary_to_integer(TS), El}];
 			   {error, _} ->
@@ -399,13 +408,13 @@ get_max_direction_id(RSM) ->
 	    {undefined, undefined, <<>>}
     end.
 
--spec make_archive_el(binary(), binary(), binary(), binary(),
+-spec make_archive_el(binary(), binary(), binary(), binary(), binary(),
 		      binary(), _, jid(), jid()) ->
 			     {ok, xmpp_element()} | {error, invalid_jid |
 						     invalid_timestamp |
 						     invalid_xml}.
-make_archive_el(TS, XML, Peer, Kind, Nick, MsgType, JidRequestor, JidArchive) ->
-    case fxml_stream:parse_element(XML) of
+make_archive_el(User, TS, XML, Peer, Kind, Nick, MsgType, JidRequestor, JidArchive) ->
+    case xml_compress:decode(XML, User, Peer) of
 	#xmlel{} = El ->
 	    try binary_to_integer(TS) of
 		TSInt ->

src/mod_muc_admin.erl

@@ -357,7 +357,7 @@ build_summary_room(Name, Host, Pid) ->
     C = get_room_config(Pid),
     Public = C#config.public,
     S = get_room_state(Pid),
-    Participants = dict:size(S#state.users),
+    Participants = maps:size(S#state.users),
     {<<Name/binary, "@", Host/binary>>,
 	 misc:atom_to_binary(Public),
      Participants
@@ -523,7 +523,7 @@ build_info_room({Name, Host, Pid}) ->
 
     S = get_room_state(Pid),
     Just_created = S#state.just_created,
-    Num_participants = length(dict:fetch_keys(S#state.users)),
+    Num_participants = maps:size(S#state.users),
 
     History = (S#state.history)#lqueue.queue,
     Ts_last_message =
@@ -778,7 +778,7 @@ decide_room({_Room_name, _Host, Room_pid}, Last_allowed) ->
     Just_created = S#state.just_created,
 
     Room_users = S#state.users,
-    Num_users = length(?DICT:to_list(Room_users)),
+    Num_users = maps:size(Room_users),
 
     History = (S#state.history)#lqueue.queue,
     Ts_now = calendar:universal_time(),
@@ -854,7 +854,7 @@ get_room_occupants(Pid) ->
 	       Info#user.nick,
 	       atom_to_list(Info#user.role)}
       end,
-      dict:to_list(S#state.users)).
+      maps:to_list(S#state.users)).
 
 get_room_occupants_number(Room, Host) ->
     case get_room_pid(Room, Host) of
@@ -862,7 +862,7 @@ get_room_occupants_number(Room, Host) ->
 	    throw({error, room_not_found});
 	Pid ->
 	    S = get_room_state(Pid),
-	    dict:size(S#state.users)
+	    maps:size(S#state.users)
     end.
 
 %%----------------------------
@@ -1039,7 +1039,7 @@ get_room_affiliations(Name, Service) ->
 	{ok, Pid} ->
 	    %% Get the PID of the online room, then request its state
 	    {ok, StateData} = p1_fsm:sync_send_all_state_event(Pid, get_state),
-	    Affiliations = ?DICT:to_list(StateData#state.affiliations),
+	    Affiliations = maps:to_list(StateData#state.affiliations),
 	    lists:map(
 	      fun({{Uname, Domain, _Res}, {Aff, Reason}}) when is_atom(Aff)->
 		      {Uname, Domain, Aff, Reason};
@@ -1173,7 +1173,7 @@ get_config_opt_name(Pos) ->
         {get_config_opt_name(Opt), element(Opt, Config)}).
 make_opts(StateData) ->
     Config = StateData#state.config,
-    Subscribers = (?DICT):fold(
+    Subscribers = maps:fold(
                     fun(_LJID, Sub, Acc) ->
                             [{Sub#subscriber.jid,
                               Sub#subscriber.nick,
@@ -1205,7 +1205,7 @@ make_opts(StateData) ->
      {captcha_whitelist,
       (?SETS):to_list((StateData#state.config)#config.captcha_whitelist)},
      {affiliations,
-      (?DICT):to_list(StateData#state.affiliations)},
+      maps:to_list(StateData#state.affiliations)},
      {subject, StateData#state.subject},
      {subject_author, StateData#state.subject_author},
      {subscribers, Subscribers}].

src/mod_muc_log.erl

@@ -887,7 +887,7 @@ get_room_occupants(RoomJIDString) ->
     MucService = RoomJID#jid.lserver,
     StateData = get_room_state(RoomName, MucService),
     [{U#user.jid, U#user.nick, U#user.role}
-     || {_, U} <- (?DICT):to_list(StateData#state.users)].
+     || U <- maps:values(StateData#state.users)].
 
 -spec get_room_state(binary(), binary()) -> mod_muc_room:state().
 

src/mod_multicast.erl

@@ -34,7 +34,8 @@
 -behaviour(gen_mod).
 
 %% API
--export([start/2, stop/1, reload/3]).
+-export([start/2, stop/1, reload/3,
+         user_send_packet/1]).
 
 %% gen_server callbacks
 -export([init/1, handle_info/2, handle_call/3,
@@ -111,6 +112,40 @@ reload(LServerS, NewOpts, OldOpts) ->
     Proc = gen_mod:get_module_proc(LServerS, ?MODULE),
     gen_server:cast(Proc, {reload, NewOpts, OldOpts}).
 
+-define(SETS, gb_sets).
+
+user_send_packet({#presence{} = Packet, C2SState} = Acc) ->
+    case xmpp:get_subtag(Packet, #addresses{}) of
+        #addresses{list = Addresses} ->
+            {ToDeliver, _Delivereds} = split_addresses_todeliver(Addresses),
+            NewState =
+                lists:foldl(
+                  fun(Address, St) ->
+                          case Address#address.jid of
+                              #jid{} = JID ->
+                                  LJID = jid:tolower(JID),
+                                  #{pres_a := PresA} = St,
+                                  A =
+                                      case Packet#presence.type of
+                                          available ->
+                                              ?SETS:add_element(LJID, PresA);
+                                          unavailable ->
+                                              ?SETS:del_element(LJID, PresA);
+                                          _ ->
+                                              PresA
+                                      end,
+                                  St#{pres_a => A};
+                              undefined ->
+                                  St
+                          end
+                  end, C2SState, ToDeliver),
+            {Packet, NewState};
+	false ->
+	    Acc
+    end;
+user_send_packet(Acc) ->
+    Acc.
+
 %%====================================================================
 %% gen_server callbacks
 %%====================================================================
@@ -125,6 +160,8 @@ init([LServerS, Opts]) ->
     try_start_loop(),
     ejabberd_router_multicast:register_route(LServerS),
     ejabberd_router:register_route(LServiceS, LServerS),
+    ejabberd_hooks:add(user_send_packet, LServerS, ?MODULE,
+		       user_send_packet, 50),
     {ok,
      #state{lservice = LServiceS, lserver = LServerS,
 	    access = Access, service_limits = SLimits}}.
@@ -189,6 +226,8 @@ handle_info({get_host, Pid}, State) ->
 handle_info(_Info, State) -> {noreply, State}.
 
 terminate(_Reason, State) ->
+    ejabberd_hooks:delete(user_send_packet, State#state.lserver, ?MODULE,
+			  user_send_packet, 50),
     ejabberd_router_multicast:unregister_route(State#state.lserver),
     ejabberd_router:unregister_route(State#state.lservice),
     ok.
@@ -826,6 +865,9 @@ add_response(RServer, Response, State) ->
 search_server_on_cache(RServer, LServerS, _LServiceS, _Maxmins)
     when RServer == LServerS ->
     route_single;
+search_server_on_cache(RServer, _LServerS, LServiceS, _Maxmins)
+    when RServer == LServiceS ->
+    route_single;
 search_server_on_cache(RServer, _LServerS, LServiceS, Maxmins) ->
     case look_server(RServer) of
         not_cached ->

src/mod_offline.erl

@@ -384,7 +384,7 @@ need_to_store(LServer, #message{type = Type} = Packet) ->
 			false ->
 			    Packet#message.body /= [];
 			unless_chat_state ->
-			    not xmpp_util:is_standalone_chat_state(Packet)
+			    not misc:is_standalone_chat_state(Packet)
 		    end
 	    end;
 	true ->
@@ -795,8 +795,8 @@ add_delay_info(Packet, LServer, TS) ->
 		_ -> TS
 	    end,
     Packet1 = xmpp:put_meta(Packet, from_offline, true),
-    xmpp_util:add_delay_info(Packet1, jid:make(LServer), NewTS,
-			     <<"Offline storage">>).
+    misc:add_delay_info(Packet1, jid:make(LServer), NewTS,
+			<<"Offline storage">>).
 
 -spec get_priority_from_presence(presence()) -> integer().
 get_priority_from_presence(#presence{priority = Prio}) ->

src/mod_offline_sql.erl

@@ -48,7 +48,7 @@ store_message(#offline_msg{us = {LUser, LServer}} = M) ->
     From = M#offline_msg.from,
     To = M#offline_msg.to,
     Packet = xmpp:set_from_to(M#offline_msg.packet, From, To),
-    NewPacket = xmpp_util:add_delay_info(
+    NewPacket = misc:add_delay_info(
 		  Packet, jid:make(LServer),
 		  M#offline_msg.timestamp,
 		  <<"Offline Storage">>),
@@ -90,8 +90,17 @@ remove_expired_messages(_LServer) ->
 remove_old_messages(Days, LServer) ->
     case ejabberd_sql:sql_query(
 	   LServer,
-	   ?SQL("DELETE FROM spool"
-                " WHERE created_at < NOW() - INTERVAL %(Days)d DAY")) of
+           fun(pgsql, _) ->
+                   ejabberd_sql:sql_query_t(
+                     ?SQL("DELETE FROM spool"
+                          " WHERE created_at <"
+                          " NOW() - INTERVAL '%(Days)d DAY'"));
+              (_, _) ->
+                   ejabberd_sql:sql_query_t(
+                     ?SQL("DELETE FROM spool"
+                          " WHERE created_at < NOW() - INTERVAL %(Days)d DAY"))
+              end)
+        of
 	{updated, N} ->
 	    ?INFO_MSG("~p message(s) deleted from offline spool", [N]);
 	_Error ->
@@ -198,7 +207,7 @@ export(_Server) ->
 	      try xmpp:decode(El, ?NS_CLIENT, [ignore_els]) of
 		  Packet ->
 		      Packet1 = xmpp:set_from_to(Packet, From, To),
-		      Packet2 = xmpp_util:add_delay_info(
+		      Packet2 = misc:add_delay_info(
 				  Packet1, jid:make(LServer),
 				  TimeStamp, <<"Offline Storage">>),
 		      XML = fxml:element_to_binary(xmpp:encode(Packet2)),

src/mod_private.erl

@@ -28,17 +28,21 @@
 -author('alexey@process-one.net').
 
 -protocol({xep, 49, '1.2'}).
+-protocol({xep, 411, '0.2.0'}).
 
 -behaviour(gen_mod).
 
 -export([start/2, stop/1, reload/3, process_sm_iq/1, import_info/0,
 	 remove_user/2, get_data/2, get_data/3, export/1,
-	 import/5, import_start/2, mod_opt_type/1, set_data/3,
-	 mod_options/1, depends/2]).
+	 import/5, import_start/2, mod_opt_type/1, set_data/2,
+	 mod_options/1, depends/2, get_sm_features/5, pubsub_publish_item/6]).
+
+-export([get_commands_spec/0, bookmarks_to_pep/2]).
 
 -include("logger.hrl").
 -include("xmpp.hrl").
 -include("mod_private.hrl").
+-include("ejabberd_commands.hrl").
 
 -define(PRIVATE_CACHE, private_cache).
 
@@ -57,16 +61,23 @@ start(Host, Opts) ->
     Mod = gen_mod:db_mod(Host, Opts, ?MODULE),
     Mod:init(Host, Opts),
     init_cache(Mod, Host, Opts),
-    ejabberd_hooks:add(remove_user, Host, ?MODULE,
-		       remove_user, 50),
-    gen_iq_handler:add_iq_handler(ejabberd_sm, Host,
-				  ?NS_PRIVATE, ?MODULE, process_sm_iq).
+    ejabberd_hooks:add(remove_user, Host, ?MODULE, remove_user, 50),
+    ejabberd_hooks:add(disco_sm_features, Host, ?MODULE, get_sm_features, 50),
+    ejabberd_hooks:add(pubsub_publish_item, Host, ?MODULE, pubsub_publish_item, 50),
+    gen_iq_handler:add_iq_handler(ejabberd_sm, Host, ?NS_PRIVATE, ?MODULE, process_sm_iq),
+    ejabberd_commands:register_commands(get_commands_spec()).
 
 stop(Host) ->
-    ejabberd_hooks:delete(remove_user, Host, ?MODULE,
-			  remove_user, 50),
-    gen_iq_handler:remove_iq_handler(ejabberd_sm, Host,
-				     ?NS_PRIVATE).
+    ejabberd_hooks:delete(remove_user, Host, ?MODULE, remove_user, 50),
+    ejabberd_hooks:delete(disco_sm_features, Host, ?MODULE, get_sm_features, 50),
+    ejabberd_hooks:delete(pubsub_publish_item, Host, ?MODULE, pubsub_publish_item, 50),
+    gen_iq_handler:remove_iq_handler(ejabberd_sm, Host, ?NS_PRIVATE),
+    case gen_mod:is_loaded_elsewhere(Host, ?MODULE) of
+	false ->
+	    ejabberd_commands:unregister_commands(get_commands_spec());
+	true ->
+	    ok
+    end.
 
 reload(Host, NewOpts, OldOpts) ->
     NewMod = gen_mod:db_mod(Host, NewOpts, ?MODULE),
@@ -78,9 +89,46 @@ reload(Host, NewOpts, OldOpts) ->
     end,
     init_cache(NewMod, Host, NewOpts).
 
+depends(_Host, _Opts) ->
+    [{mod_pubsub, soft}].
+
+mod_opt_type(db_type) -> fun(T) -> ejabberd_config:v_db(?MODULE, T) end;
+mod_opt_type(O) when O == cache_life_time; O == cache_size ->
+    fun (I) when is_integer(I), I > 0 -> I;
+        (infinity) -> infinity
+    end;
+mod_opt_type(O) when O == use_cache; O == cache_missed ->
+    fun (B) when is_boolean(B) -> B end.
+
+mod_options(Host) ->
+    [{db_type, ejabberd_config:default_db(Host, ?MODULE)},
+     {use_cache, ejabberd_config:use_cache(Host)},
+     {cache_size, ejabberd_config:cache_size(Host)},
+     {cache_missed, ejabberd_config:cache_missed(Host)},
+     {cache_life_time, ejabberd_config:cache_life_time(Host)}].
+
+-spec get_sm_features({error, stanza_error()} | empty | {result, [binary()]},
+		      jid(), jid(), binary(), binary()) ->
+			     {error, stanza_error()} | empty | {result, [binary()]}.
+get_sm_features({error, _Error} = Acc, _From, _To, _Node, _Lang) ->
+    Acc;
+get_sm_features(Acc, _From, To, <<"">>, _Lang) ->
+    case gen_mod:is_loaded(To#jid.lserver, mod_pubsub) of
+	true ->
+	    {result, [?NS_BOOKMARKS_CONVERSION_0 |
+		      case Acc of
+			  {result, Features} -> Features;
+			  empty -> []
+		      end]};
+	false ->
+	    Acc
+    end;
+get_sm_features(Acc, _From, _To, _Node, _Lang) ->
+    Acc.
+
 -spec process_sm_iq(iq()) -> iq().
 process_sm_iq(#iq{type = Type, lang = Lang,
-		  from = #jid{luser = LUser, lserver = LServer},
+		  from = #jid{luser = LUser, lserver = LServer} = From,
 		  to = #jid{luser = LUser, lserver = LServer},
 		  sub_els = [#private{sub_els = Els0}]} = IQ) ->
     case filter_xmlels(Els0) of
@@ -88,9 +136,11 @@ process_sm_iq(#iq{type = Type, lang = Lang,
 	    Txt = <<"No private data found in this query">>,
 	    xmpp:make_error(IQ, xmpp:err_bad_request(Txt, Lang));
 	Data when Type == set ->
-	    case set_data(LUser, LServer, Data) of
+	    case set_data(From, Data) of
 		ok ->
 		    xmpp:make_iq_result(IQ);
+		{error, #stanza_error{} = Err} ->
+		    xmpp:make_error(IQ, Err);
 		{error, _} ->
 		    Txt = <<"Database failure">>,
 		    Err = xmpp:err_internal_server_error(Txt, Lang),
@@ -120,12 +170,21 @@ filter_xmlels(Els) ->
 	      end
       end, Els).
 
--spec set_data(binary(), binary(), [{binary(), xmlel()}]) -> ok | {error, _}.
-set_data(LUser, LServer, Data) ->
+-spec set_data(jid(), [{binary(), xmlel()}]) -> ok | {error, _}.
+set_data(JID, Data) ->
+    set_data(JID, Data, true).
+
+-spec set_data(jid(), [{binary(), xmlel()}], boolean()) -> ok | {error, _}.
+set_data(JID, Data, Publish) ->
+    {LUser, LServer, _} = jid:tolower(JID),
     Mod = gen_mod:db_mod(LServer, ?MODULE),
     case Mod:set_data(LUser, LServer, Data) of
 	ok ->
-	    delete_cache(Mod, LUser, LServer, Data);
+	    delete_cache(Mod, LUser, LServer, Data),
+	    case Publish of
+		true -> publish_data(JID, Data);
+		false -> ok
+	    end;
 	{error, _} = Err ->
 	    Err
     end.
@@ -181,6 +240,87 @@ remove_user(User, Server) ->
     Mod:del_data(LUser, LServer),
     delete_cache(Mod, LUser, LServer, Data).
 
+%%%===================================================================
+%%% Pubsub
+%%%===================================================================
+-spec publish_data(jid(), [{binary(), xmlel()}]) -> ok | {error, stanza_error()}.
+publish_data(JID, Data) ->
+    {_, LServer, _} = LBJID = jid:remove_resource(jid:tolower(JID)),
+    case gen_mod:is_loaded(LServer, mod_pubsub) of
+	true ->
+	    case lists:keyfind(?NS_STORAGE_BOOKMARKS, 1, Data) of
+		false -> ok;
+		{_, El} ->
+		    PubOpts = [{persist_items, true},
+			       {access_model, whitelist}],
+		    case mod_pubsub:publish_item(
+			   LBJID, LServer, ?NS_STORAGE_BOOKMARKS, JID,
+			   <<>>, [El], PubOpts, all) of
+			{result, _} -> ok;
+			{error, _} = Err -> Err
+		    end
+	    end;
+	false ->
+	    ok
+    end.
+
+-spec pubsub_publish_item(binary(), binary(), jid(), jid(),
+			  binary(), [xmlel()]) -> any().
+pubsub_publish_item(LServer, ?NS_STORAGE_BOOKMARKS,
+		    #jid{luser = LUser, lserver = LServer} = From,
+		    #jid{luser = LUser, lserver = LServer},
+		    _ItemId, [Payload|_]) ->
+    set_data(From, [{?NS_STORAGE_BOOKMARKS, Payload}], false);
+pubsub_publish_item(_, _, _, _, _, _) ->
+    ok.
+
+%%%===================================================================
+%%% Commands
+%%%===================================================================
+-spec get_commands_spec() -> [ejabberd_commands()].
+get_commands_spec() ->
+    [#ejabberd_commands{name = bookmarks_to_pep, tags = [private],
+			desc = "Export private XML storage bookmarks to PEP",
+			module = ?MODULE, function = bookmarks_to_pep,
+			args = [{user, binary}, {server, binary}],
+			args_desc = ["Username", "Server"],
+			args_example = [<<"bob">>, <<"example.com">>],
+			result = {res, restuple},
+			result_desc = "Result tuple",
+			result_example = {ok, <<"Bookmarks exported">>}}].
+
+-spec bookmarks_to_pep(binary(), binary())
+      -> {ok, binary()} | {error, binary()}.
+bookmarks_to_pep(User, Server) ->
+    LUser = jid:nodeprep(User),
+    LServer = jid:nameprep(Server),
+    Mod = gen_mod:db_mod(LServer, ?MODULE),
+    Res = case use_cache(Mod, LServer) of
+	      true ->
+		  ets_cache:lookup(
+		    ?PRIVATE_CACHE, {LUser, LServer, ?NS_STORAGE_BOOKMARKS},
+		    fun() ->
+			    Mod:get_data(LUser, LServer, ?NS_STORAGE_BOOKMARKS)
+		    end);
+	      false ->
+		  Mod:get_data(LUser, LServer, ?NS_STORAGE_BOOKMARKS)
+	end,
+    case Res of
+	{ok, El} ->
+	    Data = [{?NS_STORAGE_BOOKMARKS, El}],
+	    case publish_data(jid:make(User, Server), Data) of
+		ok ->
+		    {ok, <<"Bookmarks exported to PEP node">>};
+		{error, Err} ->
+		    {error, xmpp:format_stanza_error(Err)}
+	    end;
+	_ ->
+	    {error, <<"Cannot retrieve bookmarks from private XML storage">>}
+    end.
+
+%%%===================================================================
+%%% Cache
+%%%===================================================================
 -spec delete_cache(module(), binary(), binary(), [{binary(), xmlel()}]) -> ok.
 delete_cache(Mod, LUser, LServer, Data) ->
     case use_cache(Mod, LServer) of
@@ -230,6 +370,9 @@ cache_nodes(Mod, Host) ->
 	false -> ejabberd_cluster:get_nodes()
     end.
 
+%%%===================================================================
+%%% Import/Export
+%%%===================================================================
 import_info() ->
     [{<<"private_storage">>, 4}].
 
@@ -244,21 +387,3 @@ export(LServer) ->
 import(LServer, {sql, _}, DBType, Tab, L) ->
     Mod = gen_mod:db_mod(DBType, ?MODULE),
     Mod:import(LServer, Tab, L).
-
-depends(_Host, _Opts) ->
-    [].
-
-mod_opt_type(db_type) -> fun(T) -> ejabberd_config:v_db(?MODULE, T) end;
-mod_opt_type(O) when O == cache_life_time; O == cache_size ->
-    fun (I) when is_integer(I), I > 0 -> I;
-        (infinity) -> infinity
-    end;
-mod_opt_type(O) when O == use_cache; O == cache_missed ->
-    fun (B) when is_boolean(B) -> B end.
-
-mod_options(Host) ->
-    [{db_type, ejabberd_config:default_db(Host, ?MODULE)},
-     {use_cache, ejabberd_config:use_cache(Host)},
-     {cache_size, ejabberd_config:cache_size(Host)},
-     {cache_missed, ejabberd_config:cache_missed(Host)},
-     {cache_life_time, ejabberd_config:cache_life_time(Host)}].

src/mod_pubsub.erl

@@ -65,7 +65,7 @@
 
 %% exports for console debug manual use
 -export([create_node/5, create_node/7, delete_node/3,
-    subscribe_node/5, unsubscribe_node/5, publish_item/6,
+    subscribe_node/5, unsubscribe_node/5, publish_item/6, publish_item/8,
     delete_item/4, delete_item/5, send_items/7, get_items/2, get_item/3,
     get_cached_item/2, get_configure/5, set_configure/5,
     tree_action/3, node_action/4, node_call/4]).
@@ -2990,6 +2990,7 @@ send_last_pep(From, To) ->
     Host = host(ServerHost),
     Publisher = jid:tolower(From),
     Owner = jid:remove_resource(Publisher),
+    RecipientIsOwner = jid:remove_resource(jid:tolower(To)) == Owner,
     lists:foreach(
       fun(#pubsub_node{nodeid = {_, Node}, type = Type, id = Nidx, options = Options}) ->
 	      case match_option(Options, send_last_published_item, on_sub_and_presence) of
@@ -2998,8 +2999,11 @@ send_last_pep(From, To) ->
 		      Subscribed = case get_option(Options, access_model) of
 				       open -> true;
 				       presence -> true;
-				       whitelist -> false; % subscribers are added manually
-				       authorize -> false; % likewise
+				       %% TODO: Fix the 'whitelist'/'authorize'
+				       %% cases. Currently, only node owners
+				       %% receive last PEP notifications.
+				       whitelist -> RecipientIsOwner;
+				       authorize -> RecipientIsOwner;
 				       roster ->
 					   Grps = get_option(Options, roster_groups_allowed, []),
 					   {OU, OS, _} = Owner,

src/mod_push.erl

@@ -635,7 +635,7 @@ make_summary(_Host, _Pkt, _Dir) ->
 
 -spec unwrap_carbon(stanza()) -> stanza().
 unwrap_carbon(#message{meta = #{carbon_copy := true}} = Msg) ->
-    xmpp_util:unwrap_carbon(Msg);
+    misc:unwrap_carbon(Msg);
 unwrap_carbon(Stanza) ->
     Stanza.
 

src/mod_register.erl

@@ -73,8 +73,14 @@ depends(_Host, _Opts) ->
     [].
 
 -spec stream_feature_register([xmpp_element()], binary()) -> [xmpp_element()].
-stream_feature_register(Acc, _Host) ->
-    [#feature_register{}|Acc].
+stream_feature_register(Acc, Host) ->
+    case {gen_mod:get_module_opt(Host, ?MODULE, access),
+	  gen_mod:get_module_opt(Host, ?MODULE, ip_access),
+	  gen_mod:get_module_opt(Host, ?MODULE, redirect_url)} of
+	{none, _, <<>>} -> Acc;
+	{_, none, <<>>} -> Acc;
+	{_, _, _} -> [#feature_register{}|Acc]
+    end.
 
 c2s_unauthenticated_packet(#{ip := IP, server := Server} = State,
 			   #iq{type = T, sub_els = [_]} = IQ)

src/mod_stream_mgmt.erl

@@ -40,6 +40,8 @@
 -include("logger.hrl").
 -include("p1_queue.hrl").
 
+-define(STREAM_MGMT_CACHE, stream_mgmt_cache).
+
 -define(is_sm_packet(Pkt),
 	is_record(Pkt, sm_enable) or
 	is_record(Pkt, sm_resume) or
@@ -51,7 +53,8 @@
 %%%===================================================================
 %%% API
 %%%===================================================================
-start(Host, _Opts) ->
+start(Host, Opts) ->
+    init_cache(Opts),
     ejabberd_hooks:add(c2s_init, ?MODULE, c2s_stream_init, 50),
     ejabberd_hooks:add(c2s_stream_started, Host, ?MODULE,
 		       c2s_stream_started, 50),
@@ -94,7 +97,8 @@ stop(Host) ->
     ejabberd_hooks:delete(c2s_closed, Host, ?MODULE, c2s_closed, 50),
     ejabberd_hooks:delete(c2s_terminated, Host, ?MODULE, c2s_terminated, 50).
 
-reload(_Host, _NewOpts, _OldOpts) ->
+reload(_Host, NewOpts, _OldOpts) ->
+    init_cache(NewOpts),
     ?WARNING_MSG("module ~s is reloaded, but new configuration will take "
 		 "effect for newly created client connections only", [?MODULE]).
 
@@ -284,23 +288,16 @@ c2s_terminated(#{mgmt_state := resumed, jid := JID} = State, _Reason) ->
 	   [jid:encode(JID)]),
     bounce_message_queue(),
     {stop, State};
-c2s_terminated(#{mgmt_state := MgmtState, mgmt_stanzas_in := In, sid := SID,
-		 user := U, server := S, resource := R} = State, Reason) ->
-    Result = case MgmtState of
-		 timeout ->
-		     Info = [{num_stanzas_in, In}],
-		     %% TODO: Usually, ejabberd_c2s:process_terminated/2 is
-		     %% called later in the hook chain.  We swap the order so
-		     %% that the offline info won't be purged after we stored
-		     %% it.  This should be fixed in a proper way.
-		     State1 = ejabberd_c2s:process_terminated(State, Reason),
-		     ejabberd_sm:set_offline_info(SID, U, S, R, Info),
-		     {stop, State1};
-		 _ ->
-		     State
-	     end,
+c2s_terminated(#{mgmt_state := MgmtState, mgmt_stanzas_in := In,
+		 sid := {Time, _}, jid := JID} = State, _Reason) ->
+    case MgmtState of
+	timeout ->
+	    store_stanzas_in(jid:tolower(JID), Time, In);
+	_ ->
+	    ok
+    end,
     route_unacked_stanzas(State),
-    Result;
+    State;
 c2s_terminated(State, _Reason) ->
     State.
 
@@ -641,16 +638,11 @@ inherit_session_state(#{user := U, server := S,
 	{term, {R, Time}} ->
 	    case ejabberd_sm:get_session_pid(U, S, R) of
 		none ->
-		    case ejabberd_sm:get_offline_info(Time, U, S, R) of
-			none ->
+		    case pop_stanzas_in({U, S, R}, Time) of
+			error ->
 			    {error, <<"Previous session PID not found">>};
-			Info ->
-			    case proplists:get_value(num_stanzas_in, Info) of
-				undefined ->
-				    {error, <<"Previous session timed out">>};
-				H ->
-				    {error, <<"Previous session timed out">>, H}
-			    end
+			{ok, H} ->
+			    {error, <<"Previous session timed out">>, H}
 		    end;
 		OldPID ->
 		    OldSID = {Time, OldPID},
@@ -706,7 +698,7 @@ make_resume_id(#{sid := {Time, _}, resource := Resource}) ->
 			   (state(), xmlel(), erlang:timestamp()) -> xmlel().
 add_resent_delay_info(#{lserver := LServer}, El, Time)
   when is_record(El, message); is_record(El, presence) ->
-    xmpp_util:add_delay_info(El, jid:make(LServer), Time, <<"Resent">>);
+    misc:add_delay_info(El, jid:make(LServer), Time, <<"Resent">>);
 add_resent_delay_info(_State, El, _Time) ->
     %% TODO
     El.
@@ -750,6 +742,32 @@ need_to_enqueue(#{mgmt_force_enqueue := true} = State, #xmlel{}) ->
 need_to_enqueue(State, _) ->
     {false, State}.
 
+%%%===================================================================
+%%% Cache-like storage for last handled stanzas
+%%%===================================================================
+init_cache(Opts) ->
+    ets_cache:new(?STREAM_MGMT_CACHE, cache_opts(Opts)).
+
+cache_opts(Opts) ->
+    [{max_size, gen_mod:get_opt(cache_size, Opts)},
+     {life_time, infinity}].
+
+-spec store_stanzas_in(ljid(), erlang:timestamp(), non_neg_integer()) -> boolean().
+store_stanzas_in(LJID, Time, Num) ->
+    ets_cache:insert(?STREAM_MGMT_CACHE, {LJID, Time}, Num,
+		     ejabberd_cluster:get_nodes()).
+
+-spec pop_stanzas_in(ljid(), erlang:timestamp()) -> {ok, non_neg_integer()} | error.
+pop_stanzas_in(LJID, Time) ->
+    case ets_cache:lookup(?STREAM_MGMT_CACHE, {LJID, Time}) of
+	{ok, Val} ->
+	    ets_cache:delete(?STREAM_MGMT_CACHE, {LJID, Time},
+			     ejabberd_cluster:get_nodes()),
+	    {ok, Val};
+	error ->
+	    error
+    end.
+
 %%%===================================================================
 %%% Configuration processing
 %%%===================================================================
@@ -796,6 +814,11 @@ mod_opt_type(resend_on_timeout) ->
     fun(B) when is_boolean(B) -> B;
        (if_offline) -> if_offline
     end;
+mod_opt_type(cache_size) ->
+    fun(I) when is_integer(I), I>0 -> I;
+       (unlimited) -> infinity;
+       (infinity) -> infinity
+    end;
 mod_opt_type(queue_type) ->
     fun(ram) -> ram; (file) -> file end.
 
@@ -804,5 +827,6 @@ mod_options(Host) ->
      {resume_timeout, 300},
      {max_resume_timeout, undefined},
      {ack_timeout, 60},
+     {cache_size, ejabberd_config:cache_size(Host)},
      {resend_on_timeout, false},
      {queue_type, ejabberd_config:default_queue_type(Host)}].

src/node_pep.erl

@@ -118,8 +118,7 @@ create_node(Nidx, Owner) ->
     node_flat:create_node(Nidx, Owner).
 
 delete_node(Nodes) ->
-    {result, {_, _, Result}} = node_flat:delete_node(Nodes),
-    {result, {default, Result}}.
+    node_flat:delete_node(Nodes).
 
 subscribe_node(Nidx, Sender, Subscriber, AccessModel,
 	    SendLast, PresenceSubscription, RosterGroup, Options) ->

src/node_pep_sql.erl

@@ -74,8 +74,7 @@ create_node(Nidx, Owner) ->
     {result, {default, broadcast}}.
 
 delete_node(Nodes) ->
-    {result, {_, _, Result}} = node_flat_sql:delete_node(Nodes),
-    {result, {default, Result}}.
+    node_flat_sql:delete_node(Nodes).
 
 subscribe_node(Nidx, Sender, Subscriber, AccessModel,
 	    SendLast, PresenceSubscription, RosterGroup, Options) ->

src/prosody2ejabberd.erl

@@ -169,8 +169,6 @@ convert_data(Host, "roster", User, [Data]) ->
 	  end, Data),
     lists:foreach(fun mod_roster:set_roster/1, Rosters);
 convert_data(Host, "private", User, [Data]) ->
-    LUser = jid:nodeprep(User),
-    LServer = jid:nameprep(Host),
     PrivData = lists:flatmap(
 		 fun({_TagXMLNS, Raw}) ->
 			 case deserialize(Raw) of
@@ -181,7 +179,7 @@ convert_data(Host, "private", User, [Data]) ->
 				 []
 			 end
 		 end, Data),
-    mod_private:set_data(LUser, LServer, PrivData);
+    mod_private:set_data(jid:make(User, Host), PrivData);
 convert_data(Host, "vcard", User, [Data]) ->
     LServer = jid:nameprep(Host),
     case deserialize(Data) of

src/proxy_protocol.erl

@@ -0,0 +1,184 @@
+%%%----------------------------------------------------------------------
+%%% File    : ejabberd_http.erl
+%%% Author  : Paweł Chmielowski <pawel@process-one.net>
+%%% Purpose :
+%%% Created : 27 Nov 2018 by Paweł Chmielowski <pawel@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2018   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%----------------------------------------------------------------------
+-module(proxy_protocol).
+-author("pawel@process-one.net").
+
+%% API
+-export([decode/3]).
+
+decode(SockMod, Socket, Timeout) ->
+    V = SockMod:recv(Socket, 6, Timeout),
+    case V of
+	{ok, <<"PROXY ">>} ->
+	    decode_v1(SockMod, Socket, Timeout);
+	{ok, <<16#0d, 16#0a, 16#0d, 16#0a, 16#00, 16#0d>>} ->
+	    decode_v2(SockMod, Socket, Timeout);
+	_ ->
+	    {error, eproto}
+    end.
+
+decode_v1(SockMod, Socket, Timeout) ->
+    case read_until_rn(SockMod, Socket, <<>>, false, Timeout) of
+	{error, _} = Err ->
+	    Err;
+	Val ->
+	    case binary:split(Val, <<" ">>, [global]) of
+		[<<"TCP4">>, SAddr, DAddr, SPort, DPort] ->
+		    try {inet_parse:ipv4strict_address(binary_to_list(SAddr)),
+			 inet_parse:ipv4strict_address(binary_to_list(DAddr)),
+			 binary_to_integer(SPort),
+			 binary_to_integer(DPort)}
+		    of
+			{{ok, DA}, {ok, SA}, DP, SP} ->
+			    {{SA, SP}, {DA, DP}};
+			_ ->
+			    {error, eproto}
+		    catch
+			error:badarg ->
+			    {error, eproto}
+		    end;
+		[<<"TCP6">>, SAddr, DAddr, SPort, DPort] ->
+		    try {inet_parse:ipv6strict_address(binary_to_list(SAddr)),
+			 inet_parse:ipv6strict_address(binary_to_list(DAddr)),
+			 binary_to_integer(SPort),
+			 binary_to_integer(DPort)}
+		    of
+			{{ok, DA}, {ok, SA}, DP, SP} ->
+			    {{SA, SP}, {DA, DP}};
+			_ ->
+			    {error, eproto}
+		    catch
+			error:badarg ->
+			    {error, eproto}
+		    end;
+		[<<"UNKNOWN">> | _] ->
+		    {undefined, undefined}
+	    end
+    end.
+
+decode_v2(SockMod, Socket, Timeout) ->
+    case SockMod:recv(Socket, 10, Timeout) of
+	{error, _} = Err ->
+	    Err;
+	{ok, <<16#0a, 16#51, 16#55, 16#49, 16#54, 16#0a,
+	       2:4, Command:4, Transport:8, AddrLen:16/big-unsigned-integer>>} ->
+	    case SockMod:recv(Socket, AddrLen, Timeout) of
+		{error, _} = Err ->
+		    Err;
+		{ok, Data} ->
+		    case Command of
+			0 ->
+			    case {inet:sockname(Socket), inet:peername(Socket)} of
+				{{ok, SA}, {ok, DA}} ->
+				    {SA, DA};
+				{{error, _} = E, _} ->
+				    E;
+				{_, {error, _} = E} ->
+				    E
+			    end;
+			1 ->
+			    case Transport of
+				% UNSPEC or UNIX
+				V when V == 0; V == 16#31; V == 16#32 ->
+				    {{unknown, unknown}, {unknown, unknown}};
+				% IPV4 over TCP or UDP
+				V when V == 16#11; V == 16#12 ->
+				    case Data of
+					<<D1:8, D2:8, D3:8, D4:8,
+					  S1:8, S2:8, S3:8, S4:8,
+					  DP:16/big-unsigned-integer,
+					  SP:16/big-unsigned-integer,
+					  _/binary>> ->
+					    {{{S1, S2, S3, S4}, SP},
+					     {{D1, D2, D3, D4}, DP}};
+					_ ->
+					    {error, eproto}
+				    end;
+				% IPV6 over TCP or UDP
+				V when V == 16#21; V == 16#22 ->
+				    case Data of
+					<<D1:16/big-unsigned-integer,
+					  D2:16/big-unsigned-integer,
+					  D3:16/big-unsigned-integer,
+					  D4:16/big-unsigned-integer,
+					  D5:16/big-unsigned-integer,
+					  D6:16/big-unsigned-integer,
+					  D7:16/big-unsigned-integer,
+					  D8:16/big-unsigned-integer,
+					  S1:16/big-unsigned-integer,
+					  S2:16/big-unsigned-integer,
+					  S3:16/big-unsigned-integer,
+					  S4:16/big-unsigned-integer,
+					  S5:16/big-unsigned-integer,
+					  S6:16/big-unsigned-integer,
+					  S7:16/big-unsigned-integer,
+					  S8:16/big-unsigned-integer,
+					  DP:16/big-unsigned-integer,
+					  SP:16/big-unsigned-integer,
+					  _/binary>> ->
+					    {{{S1, S2, S3, S4, S5, S6, S7, S8}, SP},
+					     {{D1, D2, D3, D4, D5, D6, D7, D8}, DP}};
+					_ ->
+					    {error, eproto}
+				    end
+			    end;
+			_ ->
+			    {error, eproto}
+		    end
+	    end;
+	<<16#0a, 16#51, 16#55, 16#49, 16#54, 16#0a, _/binary>> ->
+	    {error, eproto};
+	_ ->
+	    {error, eproto}
+    end.
+
+read_until_rn(_SockMod, _Socket, Data, _, _) when size(Data) > 107 ->
+    {error, eproto};
+read_until_rn(SockMod, Socket, Data, true, Timeout) ->
+    case SockMod:recv(Socket, 1, Timeout) of
+	{ok, <<"\n">>} ->
+	    Data;
+	{ok, <<"\r">>} ->
+	    read_until_rn(SockMod, Socket, <<Data/binary, "\r">>,
+			  true, Timeout);
+	{ok, Other} ->
+	    read_until_rn(SockMod, Socket, <<Data/binary, "\r", Other/binary>>,
+			  false, Timeout);
+	{error, _} = Err ->
+	    Err
+    end;
+read_until_rn(SockMod, Socket, Data, false, Timeout) ->
+    case SockMod:recv(Socket, 2, Timeout) of
+	{ok, <<"\r\n">>} ->
+	    Data;
+	{ok, <<Byte:8, "\r">>} ->
+	    read_until_rn(SockMod, Socket, <<Data/binary, Byte:8>>,
+			  true, Timeout);
+	{ok, Other} ->
+	    read_until_rn(SockMod, Socket, <<Data/binary, Other/binary>>,
+			  false, Timeout);
+	{error, _} = Err ->
+	    Err
+    end.

src/xml_compress.erl

@@ -0,0 +1,958 @@
+-module(xml_compress).
+-export([encode/3, decode/3]).
+
+% This file was generated by xml_compress_gen
+%
+% Rules used:
+%
+%  [{<<"eu.siacs.conversations.axolotl">>,<<"key">>,
+%    [{<<"prekey">>,[<<"true">>]},{<<"rid">>,[]}],
+%    []},
+%   {<<"jabber:client">>,<<"message">>,
+%    [{<<"from">>,[j2,{j1}]},
+%     {<<"id">>,[]},
+%     {<<"to">>,[j1,j2,{j1}]},
+%     {<<"type">>,[<<"chat">>,<<"groupchat">>,<<"normal">>]},
+%     {<<"xml:lang">>,[<<"en">>]}],
+%    []},
+%   {<<"urn:xmpp:hints">>,<<"store">>,[],[]},
+%   {<<"jabber:client">>,<<"body">>,[],
+%    [<<73,32,115,101,110,116,32,121,111,117,32,97,110,32,79,77,69,77,79,32,101,
+%       110,99,114,121,112,116,101,100,32,109,101,115,115,97,103,101,32,98,117,
+%       116,32,121,111,117,114,32,99,108,105,101,110,116,32,100,111,101,115,110,
+%       226,128,153,116,32,115,101,101,109,32,116,111,32,115,117,112,112,111,
+%       114,116,32,116,104,97,116,46,32,70,105,110,100,32,109,111,114,101,32,
+%       105,110,102,111,114,109,97,116,105,111,110,32,111,110,32,104,116,116,
+%       112,115,58,47,47,99,111,110,118,101,114,115,97,116,105,111,110,115,46,
+%       105,109,47,111,109,101,109,111>>]},
+%   {<<"urn:xmpp:sid:0">>,<<"origin-id">>,[{<<"id">>,[]}],[]},
+%   {<<"urn:xmpp:chat-markers:0">>,<<"markable">>,[],[]},
+%   {<<"eu.siacs.conversations.axolotl">>,<<"encrypted">>,[],[]},
+%   {<<"eu.siacs.conversations.axolotl">>,<<"header">>,[{<<"sid">>,[]}],[]},
+%   {<<"eu.siacs.conversations.axolotl">>,<<"iv">>,[],[]},
+%   {<<"eu.siacs.conversations.axolotl">>,<<"payload">>,[],[]},
+%   {<<"urn:xmpp:eme:0">>,<<"encryption">>,
+%    [{<<"name">>,[<<"OMEMO">>]},
+%     {<<"namespace">>,[<<"eu.siacs.conversations.axolotl">>]}],
+%    []},
+%   {<<"urn:xmpp:delay">>,<<"delay">>,[{<<"from">>,[j1]},{<<"stamp">>,[]}],[]},
+%   {<<"http://jabber.org/protocol/address">>,<<"address">>,
+%    [{<<"jid">>,[{j1}]},{<<"type">>,[<<"ofrom">>]}],
+%    []},
+%   {<<"http://jabber.org/protocol/address">>,<<"addresses">>,[],[]},
+%   {<<"urn:xmpp:chat-markers:0">>,<<"displayed">>,
+%    [{<<"id">>,[]},{<<"sender">>,[{j1},{j2}]}],
+%    []},
+%   {<<"urn:xmpp:mam:tmp">>,<<"archived">>,[{<<"by">>,[]},{<<"id">>,[]}],[]},
+%   {<<"urn:xmpp:sid:0">>,<<"stanza-id">>,[{<<"by">>,[]},{<<"id">>,[]}],[]},
+%   {<<"urn:xmpp:receipts">>,<<"request">>,[],[]},
+%   {<<"urn:xmpp:chat-markers:0">>,<<"received">>,[{<<"id">>,[]}],[]},
+%   {<<"urn:xmpp:receipts">>,<<"received">>,[{<<"id">>,[]}],[]},
+%   {<<"http://jabber.org/protocol/chatstates">>,<<"active">>,[],[]},
+%   {<<"http://jabber.org/protocol/muc#user">>,<<"invite">>,
+%    [{<<"from">>,[{j1}]}],
+%    []},
+%   {<<"http://jabber.org/protocol/muc#user">>,<<"reason">>,[],[]},
+%   {<<"http://jabber.org/protocol/muc#user">>,<<"x">>,[],[]},
+%   {<<"jabber:x:conference">>,<<"x">>,[{<<"jid">>,[j2]}],[]},
+%   {<<"jabber:client">>,<<"subject">>,[],[]},
+%   {<<"jabber:client">>,<<"thread">>,[],[]},
+%   {<<"http://jabber.org/protocol/pubsub#event">>,<<"event">>,[],[]},
+%   {<<"http://jabber.org/protocol/pubsub#event">>,<<"item">>,[{<<"id">>,[]}],[]},
+%   {<<"http://jabber.org/protocol/pubsub#event">>,<<"items">>,
+%    [{<<"node">>,[<<"urn:xmpp:mucsub:nodes:messages">>]}],
+%    []},
+%   {<<"p1:push:custom">>,<<"x">>,[{<<"key">>,[]},{<<"value">>,[]}],[]},
+%   {<<"p1:pushed">>,<<"x">>,[],[]},
+%   {<<"urn:xmpp:message-correct:0">>,<<"replace">>,[{<<"id">>,[]}],[]},
+%   {<<"http://jabber.org/protocol/chatstates">>,<<"composing">>,[],[]}]
+
+encode(El, J1, J2) ->
+  encode_child(El, <<"jabber:client">>,
+    J1, J2, byte_size(J1), byte_size(J2), <<1:8>>).
+
+encode_attr({<<"xmlns">>, _}, Acc) ->
+  Acc;
+encode_attr({N, V}, Acc) ->
+  <<Acc/binary, 1:8, (encode_string(N))/binary,
+    (encode_string(V))/binary>>.
+
+encode_attrs(Attrs, Acc) ->
+  lists:foldl(fun encode_attr/2, Acc, Attrs).
+
+encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  E1 = if
+    PNs == Ns -> encode_attrs(Attrs, <<Pfx/binary, 2:8, (encode_string(Name))/binary>>);
+    true -> encode_attrs(Attrs, <<Pfx/binary, 3:8, (encode_string(Ns))/binary, (encode_string(Name))/binary>>)
+  end,
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E1/binary, 2:8>>),
+  <<E2/binary, 4:8>>.
+
+encode_child({xmlel, Name, Attrs, Children}, PNs, J1, J2, J1L, J2L, Pfx) ->
+  case lists:keyfind(<<"xmlns">>, 1, Attrs) of
+    false ->
+      encode(PNs, PNs, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx);
+    {_, Ns} ->
+      encode(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+  end;
+encode_child({xmlcdata, Data}, _PNs, _J1, _J2, _J1L, _J2L, Pfx) ->
+  <<Pfx/binary, 1:8, (encode_string(Data))/binary>>.
+
+encode_children(Children, PNs, J1, J2, J1L, J2L, Pfx) ->
+  lists:foldl(
+    fun(Child, Acc) ->
+      encode_child(Child, PNs, J1, J2, J1L, J2L, Acc)
+    end, Pfx, Children).
+
+encode_string(Data) ->
+  <<V1:4, V2:6, V3:6>> = <<(byte_size(Data)):16/unsigned-big-integer>>,
+  case {V1, V2, V3} of
+    {0, 0, V3} ->
+      <<V3:8, Data/binary>>;
+    {0, V2, V3} ->
+      <<(V3 bor 64):8, V2:8, Data/binary>>;
+    _ ->
+      <<(V3 bor 64):8, (V2 bor 64):8, V1:8, Data/binary>>
+  end.
+
+encode(PNs, <<"eu.siacs.conversations.axolotl">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"key">> ->
+      E = lists:foldl(fun
+        ({<<"prekey">>, AVal}, Acc) ->
+          case AVal of
+            <<"true">> -> <<Acc/binary, 3:8>>;
+            _ -> <<Acc/binary, 4:8, (encode_string(AVal))/binary>>
+          end;
+        ({<<"rid">>, AVal}, Acc) ->
+          <<Acc/binary, 5:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 5:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"encrypted">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 12:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"header">> ->
+      E = lists:foldl(fun
+        ({<<"sid">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 13:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"iv">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 14:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"payload">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 15:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"jabber:client">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"message">> ->
+      E = lists:foldl(fun
+        ({<<"from">>, AVal}, Acc) ->
+          case AVal of
+            J2 -> <<Acc/binary, 3:8>>;
+            <<J1:J1L/binary, Rest/binary>> -> <<Acc/binary, 4:8, (encode_string(Rest))/binary>>;
+            _ -> <<Acc/binary, 5:8, (encode_string(AVal))/binary>>
+          end;
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 6:8, (encode_string(AVal))/binary>>;
+        ({<<"to">>, AVal}, Acc) ->
+          case AVal of
+            J1 -> <<Acc/binary, 7:8>>;
+            J2 -> <<Acc/binary, 8:8>>;
+            <<J1:J1L/binary, Rest/binary>> -> <<Acc/binary, 9:8, (encode_string(Rest))/binary>>;
+            _ -> <<Acc/binary, 10:8, (encode_string(AVal))/binary>>
+          end;
+        ({<<"type">>, AVal}, Acc) ->
+          case AVal of
+            <<"chat">> -> <<Acc/binary, 11:8>>;
+            <<"groupchat">> -> <<Acc/binary, 12:8>>;
+            <<"normal">> -> <<Acc/binary, 13:8>>;
+            _ -> <<Acc/binary, 14:8, (encode_string(AVal))/binary>>
+          end;
+        ({<<"xml:lang">>, AVal}, Acc) ->
+          case AVal of
+            <<"en">> -> <<Acc/binary, 15:8>>;
+            _ -> <<Acc/binary, 16:8, (encode_string(AVal))/binary>>
+          end;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 6:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"body">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 8:8>>),
+  E2 = lists:foldl(fun
+    ({xmlcdata, <<73,32,115,101,110,116,32,121,111,117,32,97,110,32,79,77,69,
+                  77,79,32,101,110,99,114,121,112,116,101,100,32,109,101,115,
+                  115,97,103,101,32,98,117,116,32,121,111,117,114,32,99,108,
+                  105,101,110,116,32,100,111,101,115,110,226,128,153,116,32,
+                  115,101,101,109,32,116,111,32,115,117,112,112,111,114,116,32,
+                  116,104,97,116,46,32,70,105,110,100,32,109,111,114,101,32,
+                  105,110,102,111,114,109,97,116,105,111,110,32,111,110,32,104,
+                  116,116,112,115,58,47,47,99,111,110,118,101,114,115,97,116,
+                  105,111,110,115,46,105,109,47,111,109,101,109,111>>}, Acc) -> <<Acc/binary, 9:8>>;
+    (El, Acc) -> encode_child(El, Ns, J1, J2, J1L, J2L, Acc)
+  end, <<E/binary, 2:8>>, Children),
+  <<E2/binary, 4:8>>;
+    <<"subject">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 31:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"thread">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 32:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"urn:xmpp:hints">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"store">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 7:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"urn:xmpp:sid:0">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"origin-id">> ->
+      E = lists:foldl(fun
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 10:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"stanza-id">> ->
+      E = lists:foldl(fun
+        ({<<"by">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 4:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 22:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"urn:xmpp:chat-markers:0">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"markable">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 11:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"displayed">> ->
+      E = lists:foldl(fun
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+        ({<<"sender">>, AVal}, Acc) ->
+          case AVal of
+            <<J1:J1L/binary, Rest/binary>> -> <<Acc/binary, 4:8, (encode_string(Rest))/binary>>;
+            <<J2:J2L/binary, Rest/binary>> -> <<Acc/binary, 5:8, (encode_string(Rest))/binary>>;
+            _ -> <<Acc/binary, 6:8, (encode_string(AVal))/binary>>
+          end;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 20:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"received">> ->
+      E = lists:foldl(fun
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 24:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"urn:xmpp:eme:0">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"encryption">> ->
+      E = lists:foldl(fun
+        ({<<"name">>, AVal}, Acc) ->
+          case AVal of
+            <<"OMEMO">> -> <<Acc/binary, 3:8>>;
+            _ -> <<Acc/binary, 4:8, (encode_string(AVal))/binary>>
+          end;
+        ({<<"namespace">>, AVal}, Acc) ->
+          case AVal of
+            <<"eu.siacs.conversations.axolotl">> -> <<Acc/binary, 5:8>>;
+            _ -> <<Acc/binary, 6:8, (encode_string(AVal))/binary>>
+          end;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 16:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"urn:xmpp:delay">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"delay">> ->
+      E = lists:foldl(fun
+        ({<<"from">>, AVal}, Acc) ->
+          case AVal of
+            J1 -> <<Acc/binary, 3:8>>;
+            _ -> <<Acc/binary, 4:8, (encode_string(AVal))/binary>>
+          end;
+        ({<<"stamp">>, AVal}, Acc) ->
+          <<Acc/binary, 5:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 17:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"http://jabber.org/protocol/address">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"address">> ->
+      E = lists:foldl(fun
+        ({<<"jid">>, AVal}, Acc) ->
+          case AVal of
+            <<J1:J1L/binary, Rest/binary>> -> <<Acc/binary, 3:8, (encode_string(Rest))/binary>>;
+            _ -> <<Acc/binary, 4:8, (encode_string(AVal))/binary>>
+          end;
+        ({<<"type">>, AVal}, Acc) ->
+          case AVal of
+            <<"ofrom">> -> <<Acc/binary, 5:8>>;
+            _ -> <<Acc/binary, 6:8, (encode_string(AVal))/binary>>
+          end;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 18:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"addresses">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 19:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"urn:xmpp:mam:tmp">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"archived">> ->
+      E = lists:foldl(fun
+        ({<<"by">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 4:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 21:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"urn:xmpp:receipts">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"request">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 23:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"received">> ->
+      E = lists:foldl(fun
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 25:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"http://jabber.org/protocol/chatstates">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"active">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 26:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"composing">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 39:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"http://jabber.org/protocol/muc#user">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"invite">> ->
+      E = lists:foldl(fun
+        ({<<"from">>, AVal}, Acc) ->
+          case AVal of
+            <<J1:J1L/binary, Rest/binary>> -> <<Acc/binary, 3:8, (encode_string(Rest))/binary>>;
+            _ -> <<Acc/binary, 4:8, (encode_string(AVal))/binary>>
+          end;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 27:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"reason">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 28:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"x">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 29:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"jabber:x:conference">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"x">> ->
+      E = lists:foldl(fun
+        ({<<"jid">>, AVal}, Acc) ->
+          case AVal of
+            J2 -> <<Acc/binary, 3:8>>;
+            _ -> <<Acc/binary, 4:8, (encode_string(AVal))/binary>>
+          end;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 30:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"http://jabber.org/protocol/pubsub#event">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"event">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 33:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"item">> ->
+      E = lists:foldl(fun
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 34:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+    <<"items">> ->
+      E = lists:foldl(fun
+        ({<<"node">>, AVal}, Acc) ->
+          case AVal of
+            <<"urn:xmpp:mucsub:nodes:messages">> -> <<Acc/binary, 3:8>>;
+            _ -> <<Acc/binary, 4:8, (encode_string(AVal))/binary>>
+          end;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 35:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"p1:push:custom">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"x">> ->
+      E = lists:foldl(fun
+        ({<<"key">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+        ({<<"value">>, AVal}, Acc) ->
+          <<Acc/binary, 4:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 36:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"p1:pushed">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"x">> ->
+      E = encode_attrs(Attrs, <<Pfx/binary, 37:8>>),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, <<"urn:xmpp:message-correct:0">> = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  case Name of
+    <<"replace">> ->
+      E = lists:foldl(fun
+        ({<<"id">>, AVal}, Acc) ->
+          <<Acc/binary, 3:8, (encode_string(AVal))/binary>>;
+    (Attr, Acc) -> encode_attr(Attr, Acc)
+  end, <<Pfx/binary, 38:8>>, Attrs),
+  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E/binary, 2:8>>),
+  <<E2/binary, 4:8>>;
+  _ -> encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)
+end;
+encode(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->
+  encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx).
+
+decode(<<$<, _/binary>> = Data, _J1, _J2) ->
+  fxml_stream:parse_element(Data);
+decode(<<1:8, Rest/binary>>, J1, J2) ->
+  {El, _} = decode(Rest, <<"jabber:client">>, J1, J2),
+  El.
+
+decode_string(Data) ->
+  case Data of
+    <<0:2, L:6, Str:L/binary, Rest/binary>> ->
+      {Str, Rest};
+    <<1:2, L1:6, 0:2, L2:6, Rest/binary>> ->
+      L = L2*64 + L1,
+    <<Str:L/binary, Rest2/binary>> = Rest,
+      {Str, Rest2};
+    <<1:2, L1:6, 1:2, L2:6, L3:8, Rest/binary>> ->
+      L = (L3*64 + L2)*64 + L1,
+    <<Str:L/binary, Rest2/binary>> = Rest,
+      {Str, Rest2}
+  end.
+
+decode_child(<<1:8, Rest/binary>>, _PNs, _J1, _J2) ->
+  {Text, Rest2} = decode_string(Rest),
+  {{xmlcdata, Text}, Rest2};
+decode_child(<<2:8, Rest/binary>>, PNs, J1, J2) ->
+  {Name, Rest2} = decode_string(Rest),
+  {Attrs, Rest3} = decode_attrs(Rest2),
+  {Children, Rest4} = decode_children(Rest3, PNs, J1, J2),
+  {{xmlel, Name, Attrs, Children}, Rest4};
+decode_child(<<3:8, Rest/binary>>, PNs, J1, J2) ->
+  {Name, Rest2} = decode_string(Rest),
+  {Ns, Rest3} = decode_string(Rest2),
+  {Attrs, Rest4} = decode_attrs(Rest3),
+  {Children, Rest5} = decode_children(Rest4, Ns, J1, J2),
+  {{xmlel, Name, add_ns(PNs, Ns, Attrs), Children}, Rest5};
+decode_child(<<4:8, Rest/binary>>, _PNs, _J1, _J2) ->
+  {stop, Rest};
+decode_child(Other, PNs, J1, J2) ->
+  decode(Other, PNs, J1, J2).
+
+decode_children(Data, PNs, J1, J2) ->
+  prefix_map(fun(Data2) -> decode(Data2, PNs, J1, J2) end, Data).
+
+decode_attr(<<1:8, Rest/binary>>) ->
+  {Name, Rest2} = decode_string(Rest),
+  {Val, Rest3} = decode_string(Rest2),
+  {{Name, Val}, Rest3};
+decode_attr(<<2:8, Rest/binary>>) ->
+  {stop, Rest}.
+
+decode_attrs(Data) ->
+  prefix_map(fun decode_attr/1, Data).
+
+prefix_map(F, Data) ->
+  prefix_map(F, Data, []).
+
+prefix_map(F, Data, Acc) ->
+  case F(Data) of
+    {stop, Rest} ->
+      {lists:reverse(Acc), Rest};
+    {Val, Rest} ->
+      prefix_map(F, Rest, [Val | Acc])
+  end.
+
+add_ns(Ns, Ns, Attrs) ->
+  Attrs;
+add_ns(_, Ns, Attrs) ->
+  [{<<"xmlns">>, Ns} | Attrs].
+
+decode(<<5:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"eu.siacs.conversations.axolotl">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {{<<"prekey">>, <<"true">>}, Rest3};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"prekey">>, AVal}, Rest4};
+    (<<5:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"rid">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"key">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<12:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"eu.siacs.conversations.axolotl">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"encrypted">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<13:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"eu.siacs.conversations.axolotl">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"sid">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"header">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<14:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"eu.siacs.conversations.axolotl">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"iv">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<15:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"eu.siacs.conversations.axolotl">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"payload">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<6:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"jabber:client">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {{<<"from">>, J2}, Rest3};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"from">>, <<J1/binary, AVal/binary>>}, Rest4};
+    (<<5:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"from">>, AVal}, Rest4};
+    (<<6:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<7:8, Rest3/binary>>) ->
+      {{<<"to">>, J1}, Rest3};
+    (<<8:8, Rest3/binary>>) ->
+      {{<<"to">>, J2}, Rest3};
+    (<<9:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"to">>, <<J1/binary, AVal/binary>>}, Rest4};
+    (<<10:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"to">>, AVal}, Rest4};
+    (<<11:8, Rest3/binary>>) ->
+      {{<<"type">>, <<"chat">>}, Rest3};
+    (<<12:8, Rest3/binary>>) ->
+      {{<<"type">>, <<"groupchat">>}, Rest3};
+    (<<13:8, Rest3/binary>>) ->
+      {{<<"type">>, <<"normal">>}, Rest3};
+    (<<14:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"type">>, AVal}, Rest4};
+    (<<15:8, Rest3/binary>>) ->
+      {{<<"xml:lang">>, <<"en">>}, Rest3};
+    (<<16:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"xml:lang">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"message">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<8:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"jabber:client">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = prefix_map(fun    (<<9:8, Rest5/binary>>) ->
+      {{xmlcdata, <<73,32,115,101,110,116,32,121,111,117,32,97,110,32,79,77,69,
+                    77,79,32,101,110,99,114,121,112,116,101,100,32,109,101,115,
+                    115,97,103,101,32,98,117,116,32,121,111,117,114,32,99,108,
+                    105,101,110,116,32,100,111,101,115,110,226,128,153,116,32,
+                    115,101,101,109,32,116,111,32,115,117,112,112,111,114,116,
+                    32,116,104,97,116,46,32,70,105,110,100,32,109,111,114,101,
+                    32,105,110,102,111,114,109,97,116,105,111,110,32,111,110,
+                    32,104,116,116,112,115,58,47,47,99,111,110,118,101,114,115,
+                    97,116,105,111,110,115,46,105,109,47,111,109,101,109,111>>}, Rest5};
+    (Other) ->
+      decode_child(Other, Ns, J1, J2)
+  end, Rest2),
+  {{xmlel, <<"body">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<31:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"jabber:client">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"subject">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<32:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"jabber:client">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"thread">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<7:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:hints">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"store">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<10:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:sid:0">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"origin-id">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<22:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:sid:0">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"by">>, AVal}, Rest4};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"stanza-id">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<11:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:chat-markers:0">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"markable">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<20:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:chat-markers:0">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"sender">>, <<J1/binary, AVal/binary>>}, Rest4};
+    (<<5:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"sender">>, <<J2/binary, AVal/binary>>}, Rest4};
+    (<<6:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"sender">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"displayed">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<24:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:chat-markers:0">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"received">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<16:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:eme:0">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {{<<"name">>, <<"OMEMO">>}, Rest3};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"name">>, AVal}, Rest4};
+    (<<5:8, Rest3/binary>>) ->
+      {{<<"namespace">>, <<"eu.siacs.conversations.axolotl">>}, Rest3};
+    (<<6:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"namespace">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"encryption">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<17:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:delay">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {{<<"from">>, J1}, Rest3};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"from">>, AVal}, Rest4};
+    (<<5:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"stamp">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"delay">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<18:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/address">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"jid">>, <<J1/binary, AVal/binary>>}, Rest4};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"jid">>, AVal}, Rest4};
+    (<<5:8, Rest3/binary>>) ->
+      {{<<"type">>, <<"ofrom">>}, Rest3};
+    (<<6:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"type">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"address">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<19:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/address">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"addresses">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<21:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:mam:tmp">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"by">>, AVal}, Rest4};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"archived">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<23:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:receipts">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"request">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<25:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:receipts">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"received">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<26:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/chatstates">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"active">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<39:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/chatstates">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"composing">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<27:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/muc#user">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"from">>, <<J1/binary, AVal/binary>>}, Rest4};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"from">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"invite">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<28:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/muc#user">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"reason">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<29:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/muc#user">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"x">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<30:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"jabber:x:conference">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {{<<"jid">>, J2}, Rest3};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"jid">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"x">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<33:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/pubsub#event">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"event">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<34:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/pubsub#event">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"item">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<35:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"http://jabber.org/protocol/pubsub#event">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {{<<"node">>, <<"urn:xmpp:mucsub:nodes:messages">>}, Rest3};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"node">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"items">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<36:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"p1:push:custom">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"key">>, AVal}, Rest4};
+    (<<4:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"value">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"x">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<37:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"p1:pushed">>,
+  {Attrs, Rest2} = decode_attrs(Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"x">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(<<38:8, Rest/binary>>, PNs, J1, J2) ->
+  Ns = <<"urn:xmpp:message-correct:0">>,
+  {Attrs, Rest2} = prefix_map(fun
+    (<<3:8, Rest3/binary>>) ->
+      {AVal, Rest4} = decode_string(Rest3),
+      {{<<"id">>, AVal}, Rest4};
+    (<<2:8, Rest3/binary>>) ->
+      {stop, Rest3};
+    (Data) ->
+      decode_attr(Data)
+  end, Rest),
+  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),
+  {{xmlel, <<"replace">>, add_ns(PNs, Ns, Attrs), Children}, Rest6};
+decode(Other, PNs, J1, J2) ->
+  decode_child(Other, PNs, J1, J2).
+

test/ejabberd_SUITE.erl

@@ -401,7 +401,7 @@ db_tests(riak) ->
        presence_broadcast,
        last,
        roster_tests:single_cases(),
-       private,
+       %%private_tests:single_cases(),
        privacy_tests:single_cases(),
        vcard_tests:single_cases(),
        muc_tests:single_cases(),
@@ -424,7 +424,7 @@ db_tests(DB) when DB == mnesia; DB == redis ->
        presence_broadcast,
        last,
        roster_tests:single_cases(),
-       private,
+       private_tests:single_cases(),
        privacy_tests:single_cases(),
        vcard_tests:single_cases(),
        pubsub_tests:single_cases(),
@@ -455,7 +455,7 @@ db_tests(_) ->
        presence_broadcast,
        last,
        roster_tests:single_cases(),
-       private,
+       private_tests:single_cases(),
        privacy_tests:single_cases(),
        vcard_tests:single_cases(),
        pubsub_tests:single_cases(),
@@ -602,7 +602,7 @@ test_connect_bad_ns_stream(Config) ->
 test_connect_bad_lang(Config) ->
     Lang = iolist_to_binary(lists:duplicate(36, $x)),
     Config0 = init_stream(set_opt(lang, Lang, Config)),
-    ?recv1(#stream_error{reason = 'policy-violation'}),
+    ?recv1(#stream_error{reason = 'invalid-xml'}),
     ?recv1({xmlstreamend, <<"stream:stream">>}),
     close_socket(Config0).
 
@@ -978,33 +978,6 @@ disco(Config) ->
       end, Items),
     disconnect(Config).
 
-private(Config) ->
-    Conference = #bookmark_conference{name = <<"Some name">>,
-                                      autojoin = true,
-                                      jid = jid:make(
-                                              <<"some">>,
-                                              <<"some.conference.org">>,
-                                              <<>>)},
-    Storage = #bookmark_storage{conference = [Conference]},
-    StorageXMLOut = xmpp:encode(Storage),
-    WrongEl = #xmlel{name = <<"wrong">>},
-    #iq{type = error} =
-        send_recv(Config, #iq{type = get,
-			      sub_els = [#private{sub_els = [WrongEl]}]}),
-    #iq{type = result, sub_els = []} =
-        send_recv(
-          Config, #iq{type = set,
-                      sub_els = [#private{sub_els = [WrongEl, StorageXMLOut]}]}),
-    #iq{type = result,
-        sub_els = [#private{sub_els = [StorageXMLIn]}]} =
-        send_recv(
-          Config,
-          #iq{type = get,
-              sub_els = [#private{sub_els = [xmpp:encode(
-                                               #bookmark_storage{})]}]}),
-    Storage = xmpp:decode(StorageXMLIn),
-    disconnect(Config).
-
 last(Config) ->
     true = is_feature_advertised(Config, ?NS_LAST),
     #iq{type = result, sub_els = [#last{}]} =

test/mod_http_api_test.exs

@@ -61,15 +61,15 @@ defmodule ModHttpApiTest do
   test "Attempting to access a command that is not exposed as HTTP API returns 403" do
     setup_mocks()
     assert :ok == :ejabberd_commands.expose_commands([])
-    request = request(method: :POST, ip: {{127,0,0,1},50000}, data: "[]")
+    request = request(method: :POST, ip: {{127,0,0,1},50000}, data: "{}")
     {403, _, _} = :mod_http_api.process(["open_cmd"], request)
   end
 
   test "Call to user, admin or restricted commands without authentication are rejected" do
     setup_mocks()
     assert :ok == :ejabberd_commands.expose_commands([:user_cmd, :admin_cmd, :restricted])
-    request = request(method: :POST, ip: {{127,0,0,1},50000}, data: "[]")
-    {403, _, _} = :mod_http_api.process(["user_cmd"], request)
+    request = request(method: :POST, ip: {{127,0,0,1},50000}, data: "{}")
+    {400, _, _} = :mod_http_api.process(["user_cmd"], request)
     {403, _, _} = :mod_http_api.process(["admin_cmd"], request)
     {403, _, _} = :mod_http_api.process(["restricted_cmd"], request)
   end

test/private_tests.erl

@@ -0,0 +1,105 @@
+%%%-------------------------------------------------------------------
+%%% Author  : Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% Created : 23 Nov 2018 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2018   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(private_tests).
+
+%% API
+-compile(export_all).
+-import(suite, [my_jid/1, is_feature_advertised/3,
+		send_recv/2, disconnect/1]).
+
+-include("suite.hrl").
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+%%%===================================================================
+%%% Single user tests
+%%%===================================================================
+single_cases() ->
+    {private_single, [sequence],
+     [single_test(test_features),
+      single_test(test_no_namespace),
+      single_test(test_set_get),
+      single_test(test_published)]}.
+
+test_features(Config) ->
+    MyJID = my_jid(Config),
+    true = is_feature_advertised(Config, ?NS_BOOKMARKS_CONVERSION_0,
+				 jid:remove_resource(MyJID)),
+    disconnect(Config).
+
+test_no_namespace(Config) ->
+    WrongEl = #xmlel{name = <<"wrong">>},
+    #iq{type = error} =
+        send_recv(Config, #iq{type = get,
+			      sub_els = [#private{sub_els = [WrongEl]}]}),
+    disconnect(Config).
+
+test_set_get(Config) ->
+    Storage = bookmark_storage(),
+    StorageXMLOut = xmpp:encode(Storage),
+    #iq{type = result, sub_els = []} =
+        send_recv(
+          Config, #iq{type = set,
+                      sub_els = [#private{sub_els = [StorageXMLOut]}]}),
+    #iq{type = result,
+        sub_els = [#private{sub_els = [StorageXMLIn]}]} =
+        send_recv(
+          Config,
+          #iq{type = get,
+              sub_els = [#private{sub_els = [xmpp:encode(
+                                               #bookmark_storage{})]}]}),
+    Storage = xmpp:decode(StorageXMLIn),
+    disconnect(Config).
+
+test_published(Config) ->
+    Storage = bookmark_storage(),
+    Node = xmpp:get_ns(Storage),
+    #iq{type = result,
+	sub_els = [#pubsub{items = #ps_items{node = Node, items = Items}}]} =
+	send_recv(
+	  Config,
+	  #iq{type = get,
+	      sub_els = [#pubsub{items = #ps_items{node = Node}}]}),
+    [#ps_item{sub_els = [StorageXMLIn]}] = Items,
+    Storage = xmpp:decode(StorageXMLIn),
+    #iq{type = result, sub_els = []} =
+	send_recv(Config,
+		  #iq{type = set,
+		      sub_els = [#pubsub_owner{delete = {Node, <<>>}}]}),
+    disconnect(Config).
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+single_test(T) ->
+    list_to_atom("private_" ++ atom_to_list(T)).
+
+conference_bookmark() ->
+    #bookmark_conference{
+       name = <<"Some name">>,
+       autojoin = true,
+       jid = jid:make(<<"some">>, <<"some.conference.org">>)}.
+
+bookmark_storage() ->
+    #bookmark_storage{conference = [conference_bookmark()]}.

tools/xml_compress_gen.erl

@@ -0,0 +1,417 @@
+%% File    : xml_compress_gen.erl
+%% Author  : Pawel Chmielowski
+%% Purpose :
+%% Created :  14 Sep 2018 Pawel Chmielowski
+%%
+%%
+%% ejabberd, Copyright (C) 2002-2018  ProcessOne
+%%
+%% This program is free software; you can redistribute it and/or
+%% modify it under the terms of the GNU General Public License as
+%% published by the Free Software Foundation; either version 2 of the
+%% License, or (at your option) any later version.
+%%
+%% This program is distributed in the hope that it will be useful,
+%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%% General Public License for more details.
+%%
+%% You should have received a copy of the GNU General Public License along
+%% with this program; if not, write to the Free Software Foundation, Inc.,
+%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%
+
+-module(xml_compress_gen).
+-author("pawel@process-one.net").
+
+-include("xmpp.hrl").
+
+%% API
+-export([archive_analyze/3, process_stats/1, gen_code/3]).
+
+-record(el_stats, {count = 0, empty_count = 0, only_text_count = 0, attrs = #{}, text_stats = #{}}).
+-record(attr_stats, {count = 0, vals = #{}}).
+
+archive_analyze(Host, Table, EHost) ->
+    case ejabberd_sql:sql_query(Host, <<"select username, peer, kind, xml from ", Table/binary>>) of
+	{selected, _, Res} ->
+	    lists:foldl(
+		fun([U, P, K, X], Stats) ->
+		    M = case K of
+			    <<"groupchat">> ->
+				U;
+			    _ ->
+				<<U/binary, "@", EHost/binary>>
+			end,
+		    El = fxml_stream:parse_element(X),
+		    analyze_element({El, <<"stream">>, <<"jabber:client">>, M, P}, Stats)
+		end, {0, #{}}, Res);
+	_ ->
+	    none
+    end.
+
+encode_id(Num) when Num < 64 ->
+    iolist_to_binary(io_lib:format("~p:8", [Num])).
+
+gen_code(_File, _Rules, $<) ->
+    {error, <<"Invalid version">>};
+gen_code(File, Rules, Ver) when Ver < 64 ->
+    {Data, _} = lists:foldl(
+	     fun({Ns, El, Attrs, Text}, {Acc, Id}) ->
+	    NsC = case lists:keyfind(Ns, 1, Acc) of
+		      false -> [];
+		      {_, L} -> L
+		  end,
+	    {AttrsE, _} = lists:mapfoldl(
+		fun({AName, AVals}, Id2) ->
+		    {AD, Id3} = lists:mapfoldl(
+			fun(AVal, Id3) ->
+			    {{AVal, encode_id(Id3)}, Id3 + 1}
+			end, Id2, AVals),
+		    {{AName, AD ++ [encode_id(Id3)]}, Id3 + 1}
+		end, 3, Attrs),
+	    {TextE, Id5} = lists:mapfoldl(
+		fun(TextV, Id4) ->
+		    {{TextV, encode_id(Id4)}, Id4 + 1}
+		end, Id + 1, Text),
+	    {lists:keystore(Ns, 1, Acc, {Ns, NsC ++ [{El, encode_id(Id), AttrsE, TextE}]}), Id5}
+	end, {[], 5}, Rules),
+    {ok, Dev} = file:open(File, write),
+    Mod = filename:basename(File, ".erl"),
+    io:format(Dev, "-module(~s).~n-export([encode/3, decode/3]).~n~n", [Mod]),
+    RulesS = iolist_to_binary(io_lib:format("~p", [Rules])),
+    RulesS2 = binary:replace(RulesS, <<"\n">>, <<"\n%  ">>, [global]),
+    io:format(Dev, "% This file was generated by xml_compress_gen~n%~n"
+		   "% Rules used:~n%~n%  ~s~n~n", [RulesS2]),
+    VerId = iolist_to_binary(io_lib:format("~p:8", [Ver])),
+    gen_encode(Dev, Data, VerId),
+    gen_decode(Dev, Data, VerId),
+    file:close(Dev),
+    Data.
+
+gen_decode(Dev, Data, VerId) ->
+    io:format(Dev, "decode(<<$<, _/binary>> = Data, _J1, _J2) ->~n"
+		   "  fxml_stream:parse_element(Data);~n"
+		   "decode(<<~s, Rest/binary>>, J1, J2) ->~n"
+		   "  {El, _} = decode(Rest, <<\"jabber:client\">>, J1, J2),~n"
+		   "  El.~n~n", [VerId]),
+    io:format(Dev, "decode_string(Data) ->~n"
+		   "  case Data of~n"
+		   "    <<0:2, L:6, Str:L/binary, Rest/binary>> ->~n"
+		   "      {Str, Rest};~n"
+		   "    <<1:2, L1:6, 0:2, L2:6, Rest/binary>> ->~n"
+		   "      L = L2*64 + L1,~n"
+		   "    <<Str:L/binary, Rest2/binary>> = Rest,~n"
+		   "      {Str, Rest2};~n"
+		   "    <<1:2, L1:6, 1:2, L2:6, L3:8, Rest/binary>> ->~n"
+		   "      L = (L3*64 + L2)*64 + L1,~n"
+		   "    <<Str:L/binary, Rest2/binary>> = Rest,~n"
+		   "      {Str, Rest2}~n"
+		   "  end.~n~n", []),
+    io:format(Dev, "decode_child(<<1:8, Rest/binary>>, _PNs, _J1, _J2) ->~n"
+		   "  {Text, Rest2} = decode_string(Rest),~n"
+		   "  {{xmlcdata, Text}, Rest2};~n", []),
+    io:format(Dev, "decode_child(<<2:8, Rest/binary>>, PNs, J1, J2) ->~n"
+		   "  {Name, Rest2} = decode_string(Rest),~n"
+		   "  {Attrs, Rest3} = decode_attrs(Rest2),~n"
+		   "  {Children, Rest4} = decode_children(Rest3, PNs, J1, J2),~n"
+		   "  {{xmlel, Name, Attrs, Children}, Rest4};~n", []),
+    io:format(Dev, "decode_child(<<3:8, Rest/binary>>, PNs, J1, J2) ->~n"
+		   "  {Name, Rest2} = decode_string(Rest),~n"
+		   "  {Ns, Rest3} = decode_string(Rest2),~n"
+		   "  {Attrs, Rest4} = decode_attrs(Rest3),~n"
+		   "  {Children, Rest5} = decode_children(Rest4, Ns, J1, J2),~n"
+		   "  {{xmlel, Name, add_ns(PNs, Ns, Attrs), Children}, Rest5};~n", []),
+    io:format(Dev, "decode_child(<<4:8, Rest/binary>>, _PNs, _J1, _J2) ->~n"
+		   "  {stop, Rest};~n", []),
+    io:format(Dev, "decode_child(Other, PNs, J1, J2) ->~n"
+		   "  decode(Other, PNs, J1, J2).~n~n", []),
+    io:format(Dev, "decode_children(Data, PNs, J1, J2) ->~n"
+		   "  prefix_map(fun(Data2) -> decode(Data2, PNs, J1, J2) end, Data).~n~n", []),
+    io:format(Dev, "decode_attr(<<1:8, Rest/binary>>) ->~n"
+		   "  {Name, Rest2} = decode_string(Rest),~n"
+		   "  {Val, Rest3} = decode_string(Rest2),~n"
+		   "  {{Name, Val}, Rest3};~n", []),
+    io:format(Dev, "decode_attr(<<2:8, Rest/binary>>) ->~n"
+		   "  {stop, Rest}.~n~n", []),
+    io:format(Dev, "decode_attrs(Data) ->~n"
+		   "  prefix_map(fun decode_attr/1, Data).~n~n", []),
+    io:format(Dev, "prefix_map(F, Data) ->~n"
+		   "  prefix_map(F, Data, []).~n~n", []),
+    io:format(Dev, "prefix_map(F, Data, Acc) ->~n"
+		   "  case F(Data) of~n"
+		   "    {stop, Rest} ->~n"
+		   "      {lists:reverse(Acc), Rest};~n"
+		   "    {Val, Rest} ->~n"
+		   "      prefix_map(F, Rest, [Val | Acc])~n"
+		   "  end.~n~n", []),
+    io:format(Dev, "add_ns(Ns, Ns, Attrs) ->~n"
+		   "  Attrs;~n"
+		   "add_ns(_, Ns, Attrs) ->~n"
+		   "  [{<<\"xmlns\">>, Ns} | Attrs].~n~n", []),
+    lists:foreach(
+	fun({Ns, Els}) ->
+	    lists:foreach(
+		fun({Name, Id, Attrs, Text}) ->
+		    io:format(Dev, "decode(<<~s, Rest/binary>>, PNs, J1, J2) ->~n"
+				   "  Ns = ~p,~n", [Id, Ns]),
+		    case Attrs of
+			[] ->
+			    io:format(Dev, "  {Attrs, Rest2} = decode_attrs(Rest),~n", []);
+			_ ->
+			    io:format(Dev, "  {Attrs, Rest2} = prefix_map(fun~n", []),
+			    lists:foreach(
+				fun({AName, AVals}) ->
+				    lists:foreach(
+					fun({j1, AId}) ->
+					    io:format(Dev, "    (<<~s, Rest3/binary>>) ->~n"
+							   "      {{~p, J1}, Rest3};~n", [AId, AName]);
+					   ({j2, AId}) ->
+					       io:format(Dev, "    (<<~s, Rest3/binary>>) ->~n"
+							      "      {{~p, J2}, Rest3};~n", [AId, AName]);
+					   ({{j1}, AId}) ->
+					       io:format(Dev, "    (<<~s, Rest3/binary>>) ->~n"
+							      "      {AVal, Rest4} = decode_string(Rest3),~n"
+							      "      {{~p, <<J1/binary, AVal/binary>>}, Rest4};~n",
+							 [AId, AName]);
+					   ({{j2}, AId}) ->
+					       io:format(Dev, "    (<<~s, Rest3/binary>>) ->~n"
+							      "      {AVal, Rest4} = decode_string(Rest3),~n"
+							      "      {{~p, <<J2/binary, AVal/binary>>}, Rest4};~n",
+							 [AId, AName]);
+					   ({AVal, AId}) ->
+					       io:format(Dev, "    (<<~s, Rest3/binary>>) ->~n"
+							      "      {{~p, ~p}, Rest3};~n",
+							 [AId, AName, AVal]);
+					   (AId) ->
+					       io:format(Dev, "    (<<~s, Rest3/binary>>) ->~n"
+							      "      {AVal, Rest4} = decode_string(Rest3),~n"
+							      "      {{~p, AVal}, Rest4};~n",
+							 [AId, AName])
+					end, AVals)
+				end, Attrs),
+			    io:format(Dev, "    (<<2:8, Rest3/binary>>) ->~n"
+					   "      {stop, Rest3};~n"
+					   "    (Data) ->~n"
+					   "      decode_attr(Data)~n"
+					   "  end, Rest),~n", [])
+		    end,
+		    case Text of
+			[] ->
+			    io:format(Dev, "  {Children, Rest6} = decode_children(Rest2, Ns, J1, J2),~n", []);
+			_ ->
+			    io:format(Dev, "  {Children, Rest6} = prefix_map(fun", []),
+			    lists:foreach(
+				fun({TextS, TId}) ->
+				    io:format(Dev, "    (<<~s, Rest5/binary>>) ->~n"
+						   "      {{xmlcdata, ~p}, Rest5};~n",
+					      [TId, TextS])
+				end, Text),
+
+			    io:format(Dev, "    (Other) ->~n"
+					   "      decode_child(Other, Ns, J1, J2)~n"
+					   "  end, Rest2),~n", [])
+		    end,
+		    io:format(Dev, "  {{xmlel, ~p, add_ns(PNs, Ns, Attrs), Children}, Rest6};~n", [Name])
+		end, Els)
+	end, Data),
+    io:format(Dev, "decode(Other, PNs, J1, J2) ->~n"
+		   "  decode_child(Other, PNs, J1, J2).~n~n", []).
+
+
+gen_encode(Dev, Data, VerId) ->
+    io:format(Dev, "encode(El, J1, J2) ->~n"
+		   "  encode_child(El, <<\"jabber:client\">>,~n"
+		   "    J1, J2, byte_size(J1), byte_size(J2), <<~s>>).~n~n", [VerId]),
+    io:format(Dev, "encode_attr({<<\"xmlns\">>, _}, Acc) ->~n"
+		   "  Acc;~n"
+		   "encode_attr({N, V}, Acc) ->~n"
+		   "  <<Acc/binary, 1:8, (encode_string(N))/binary,~n"
+		   "    (encode_string(V))/binary>>.~n~n", []),
+    io:format(Dev, "encode_attrs(Attrs, Acc) ->~n"
+		   "  lists:foldl(fun encode_attr/2, Acc, Attrs).~n~n", []),
+    io:format(Dev, "encode_el(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->~n"
+		   "  E1 = if~n"
+		   "    PNs == Ns -> encode_attrs(Attrs, <<Pfx/binary, 2:8, (encode_string(Name))/binary>>);~n"
+		   "    true -> encode_attrs(Attrs, <<Pfx/binary, 3:8, "
+		   "(encode_string(Ns))/binary, (encode_string(Name))/binary>>)~n"
+		   "  end,~n"
+		   "  E2 = encode_children(Children, Ns, J1, J2, J1L, J2L, <<E1/binary, 2:8>>),~n"
+		   "  <<E2/binary, 4:8>>.~n~n", []),
+    io:format(Dev, "encode_child({xmlel, Name, Attrs, Children}, PNs, J1, J2, J1L, J2L, Pfx) ->~n"
+		   "  case lists:keyfind(<<\"xmlns\">>, 1, Attrs) of~n"
+		   "    false ->~n"
+		   "      encode(PNs, PNs, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx);~n"
+		   "    {_, Ns} ->~n"
+		   "      encode(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx)~n"
+		   "  end;~n"
+		   "encode_child({xmlcdata, Data}, _PNs, _J1, _J2, _J1L, _J2L, Pfx) ->~n"
+		   "  <<Pfx/binary, 1:8, (encode_string(Data))/binary>>.~n~n", []),
+    io:format(Dev, "encode_children(Children, PNs, J1, J2, J1L, J2L, Pfx) ->~n"
+		   "  lists:foldl(~n"
+		   "    fun(Child, Acc) ->~n"
+		   "      encode_child(Child, PNs, J1, J2, J1L, J2L, Acc)~n"
+		   "    end, Pfx, Children).~n~n", []),
+    io:format(Dev, "encode_string(Data) ->~n"
+		   "  <<V1:4, V2:6, V3:6>> = <<(byte_size(Data)):16/unsigned-big-integer>>,~n"
+		   "  case {V1, V2, V3} of~n"
+		   "    {0, 0, V3} ->~n"
+		   "      <<V3:8, Data/binary>>;~n"
+		   "    {0, V2, V3} ->~n"
+		   "      <<(V3 bor 64):8, V2:8, Data/binary>>;~n"
+		   "    _ ->~n"
+		   "      <<(V3 bor 64):8, (V2 bor 64):8, V1:8, Data/binary>>~n"
+		   "  end.~n~n", []),
+    lists:foreach(
+	fun({Ns, Els}) ->
+	    io:format(Dev, "encode(PNs, ~p = Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->~n"
+			   "  case Name of~n", [Ns]),
+	    lists:foreach(
+		fun({ElN, Id, Attrs, Text}) ->
+		    io:format(Dev, "    ~p ->~n", [ElN]),
+		    case Attrs of
+			[] ->
+			    io:format(Dev, "      E = encode_attrs(Attrs, <<Pfx/binary, ~s>>),~n", [Id]);
+			_ ->
+			    io:format(Dev, "      E = lists:foldl(fun~n", []),
+			    lists:foreach(
+				fun({AName, AVals}) ->
+				    case AVals of
+					[AIdS] when is_binary(AIdS) ->
+					    io:format(Dev, "        ({~p, AVal}, Acc) ->~n"
+							   "          <<Acc/binary, ~s, (encode_string(AVal))/binary>>;~n",
+						      [AName, AIdS]);
+					_ ->
+					    io:format(Dev, "        ({~p, AVal}, Acc) ->~n"
+							   "          case AVal of~n", [AName]),
+					    lists:foreach(
+						fun({j1, AId}) ->
+						    io:format(Dev, "            J1 -> <<Acc/binary, ~s>>;~n",
+							      [AId]);
+						   ({j2, AId}) ->
+						       io:format(Dev, "            J2 -> <<Acc/binary, ~s>>;~n",
+								 [AId]);
+						   ({{j1}, AId}) ->
+						       io:format(Dev, "            <<J1:J1L/binary, Rest/binary>> -> "
+								      "<<Acc/binary, ~s, (encode_string(Rest))/binary>>;~n",
+								 [AId]);
+						   ({{j2}, AId}) ->
+						       io:format(Dev, "            <<J2:J2L/binary, Rest/binary>> -> "
+								      "<<Acc/binary, ~s, (encode_string(Rest))/binary>>;~n",
+								 [AId]);
+						   ({AVal, AId}) ->
+						       io:format(Dev, "            ~p -> <<Acc/binary, ~s>>;~n",
+								 [AVal, AId]);
+						   (AId) ->
+						       io:format(Dev, "            _ -> <<Acc/binary, ~s, "
+								      "(encode_string(AVal))/binary>>~n",
+								 [AId])
+						end, AVals),
+					    io:format(Dev, "          end;~n", [])
+				    end
+				end, Attrs),
+			    io:format(Dev, "    (Attr, Acc) -> encode_attr(Attr, Acc)~n", []),
+			    io:format(Dev, "  end, <<Pfx/binary, ~s>>, Attrs),~n", [Id])
+		    end,
+		    case Text of
+			[] ->
+			    io:format(Dev, "  E2 = encode_children(Children, Ns, "
+					   "J1, J2, J1L, J2L, <<E/binary, 2:8>>),~n", []);
+			_ ->
+			    io:format(Dev, "  E2 = lists:foldl(fun~n", []),
+			    lists:foreach(
+				fun({TextV, TId}) ->
+				    io:format(Dev, "    ({xmlcdata, ~p}, Acc) -> <<Acc/binary, ~s>>;~n", [TextV, TId])
+				end, Text),
+			    io:format(Dev, "    (El, Acc) -> encode_child(El, Ns, J1, J2, J1L, J2L, Acc)~n", []),
+			    io:format(Dev, "  end, <<E/binary, 2:8>>, Children),~n", [])
+		    end,
+		    io:format(Dev, "  <<E2/binary, 4:8>>;~n", [])
+		end, Els),
+	    io:format(Dev, "  _ -> encode_el(PNs, Ns, Name, Attrs, Children, "
+			   "J1, J2, J1L, J2L, Pfx)~nend;~n", [])
+	end, Data),
+    io:format(Dev, "encode(PNs, Ns, Name, Attrs, Children, J1, J2, J1L, J2L, Pfx) ->~n"
+		   "  encode_el(PNs, Ns, Name, Attrs, Children, "
+		   "J1, J2, J1L, J2L, Pfx).~n~n", []).
+
+process_stats({_Counts, Stats}) ->
+    SStats = lists:sort(
+	fun({_, #el_stats{count = C1}}, {_, #el_stats{count = C2}}) ->
+	    C1 >= C2
+	end, maps:to_list(Stats)),
+    lists:map(
+	fun({Name, #el_stats{count = C, attrs = A, text_stats = T}}) ->
+	    [Ns, El] = binary:split(Name, <<"<">>),
+	    Attrs = lists:filtermap(
+		fun({AN, #attr_stats{count = AC, vals = AV}}) ->
+		    if
+			AC*5 < C ->
+			    false;
+			true ->
+			    AVC = AC div min(maps:size(AV)*2, 10),
+			    AVA = [N || {N, C2} <- maps:to_list(AV), C2 > AVC],
+			    {true, {AN, AVA}}
+		    end
+		end, maps:to_list(A)),
+	    Text = [TE || {TE, TC} <- maps:to_list(T), TC > C/2],
+	    {Ns, El, Attrs, Text}
+	end, SStats).
+
+analyze_elements(Elements, Stats, PName, PNS, J1, J2) ->
+    lists:foldl(fun analyze_element/2, Stats, lists:map(fun(V) -> {V, PName, PNS, J1, J2} end, Elements)).
+
+maps_update(Key, F, InitVal, Map) ->
+    case maps:is_key(Key, Map) of
+	true ->
+	    maps:update_with(Key, F, Map);
+	_ ->
+	    maps:put(Key, F(InitVal), Map)
+    end.
+
+analyze_element({{xmlcdata, Data}, PName, PNS, _J1, _J2}, {ElCount, Stats}) ->
+    Stats2 = maps_update(<<PNS/binary, "<", PName/binary>>,
+	fun(#el_stats{text_stats = TS} = E) ->
+	    TS2 = maps_update(Data, fun(C) -> C + 1 end, 0, TS),
+	    E#el_stats{text_stats = TS2}
+	end, #el_stats{}, Stats),
+    {ElCount, Stats2};
+analyze_element({#xmlel{name = Name, attrs = Attrs, children = Children}, _PName, PNS, J1, J2}, {ElCount, Stats}) ->
+    XMLNS = case lists:keyfind(<<"xmlns">>, 1, Attrs) of
+		{_, NS} ->
+		    NS;
+		false ->
+		    PNS
+	    end,
+    NStats = maps_update(<<XMLNS/binary, "<", Name/binary>>,
+	fun(#el_stats{count = C, empty_count = EC, only_text_count = TC, attrs = A} = ES) ->
+	    A2 = lists:foldl(
+		fun({<<"xmlns">>, _}, AMap) ->
+		    AMap;
+		   ({AName, AVal}, AMap) ->
+		       J1S = size(J1),
+		       J2S = size(J2),
+		       AVal2 = case AVal of
+				   J1 ->
+				       j1;
+				   J2 ->
+				       j2;
+				   <<J1:J1S/binary, _Rest/binary>> ->
+				       {j1};
+				   <<J2:J2S/binary, _Rest/binary>> ->
+				       {j2};
+				   Other ->
+				       Other
+			       end,
+		       maps_update(AName, fun(#attr_stats{count = AC, vals = AV}) ->
+			   AV2 = maps_update(AVal2, fun(C2) -> C2 + 1 end, 0, AV),
+			   #attr_stats{count = AC + 1, vals = AV2}
+					  end, #attr_stats{}, AMap)
+		end, A, Attrs),
+	    ES#el_stats{count = C + 1,
+			empty_count = if Children == [] -> EC + 1; true ->
+			    EC end,
+			only_text_count = case Children of [{xmlcdata, _}] -> TC + 1; _ -> TC end,
+			attrs = A2}
+	end, #el_stats{}, Stats),
+    analyze_elements(Children, {ElCount + 1, NStats}, Name, XMLNS, J1, J2).