Rename aux.erl as misc.erl

Thanks Microsoft Windows to not support some filenames

configure.ac

@@ -66,7 +66,7 @@ if test "x$MAKE" = "x"; then
 fi
 
 # Change default prefix
-AC_PREFIX_DEFAULT(/)
+AC_PREFIX_DEFAULT(/usr/local)
 
 AC_ARG_ENABLE(hipe,
 [AC_HELP_STRING([--enable-hipe], [compile natively with HiPE, not recommended (default: no)])],

ejabberd.yml.example

@@ -106,6 +106,17 @@ hosts:
 ###.  ===============
 ###'  LISTENING PORTS
 
+## Define common macros used by listeners
+## define_macro:
+##   'CERTFILE': "/path/to/xmpp.pem"
+##   'CIPHERS': "ECDH:DH:!3DES:!aNULL:!eNULL:!MEDIUM@STRENGTH"
+##   'TLSOPTS':
+##     - "no_sslv3"
+##     - "no_tlsv1"
+##     - "cipher_server_preference"
+##     - "no_compression"
+##   'DHFILE': "/path/to/dhparams.pem" # generated with: openssl dhparam -out dhparams.pem 2048
+
 ##
 ## listen: The ports ejabberd will listen on, which service each is handled
 ## by and what options to start it with.
@@ -113,36 +124,53 @@ hosts:
 listen: 
   - 
     port: 5222
+    ip: "::"
     module: ejabberd_c2s
     ##
     ## If TLS is compiled in and you installed a SSL
     ## certificate, specify the full path to the
     ## file and uncomment these lines:
     ##
-    ## certfile: "/path/to/ssl.pem"
     ## starttls: true
+    ## certfile: 'CERTFILE'
+    ## protocol_options: 'TLSOPTS'
+    ## dhfile: 'DHFILE'
+    ## ciphers: 'CIPHERS'
     ##
     ## To enforce TLS encryption for client connections,
     ## use this instead of the "starttls" option:
     ##
     ## starttls_required: true
     ##
-    ## Custom OpenSSL options
+    ## Stream compression
+    ##
+    ## zlib: true
     ##
-    ## protocol_options:
-    ##   - "no_sslv3"
-    ##   - "no_tlsv1"
     max_stanza_size: 65536
     shaper: c2s_shaper
     access: c2s
   - 
     port: 5269
+    ip: "::"
     module: ejabberd_s2s_in
+  - 
+    port: 5280
+    ip: "::"
+    module: ejabberd_http
+    request_handlers:
+      "/websocket": ejabberd_http_ws
+      "/api": mod_http_api
+    ##  "/pub/archive": mod_http_fileserver
+    web_admin: true
+    http_bind: true
+    ## register: true
+    captcha: true
   ##
   ## ejabberd_service: Interact with external components (transports, ...)
   ##
   ## - 
   ##   port: 8888
+  ##   ip: "::"
   ##   module: ejabberd_service
   ##   access: all
   ##   shaper_rule: fast
@@ -175,19 +203,28 @@ listen:
   ##
   ## - 
   ##   port: 4560
+  ##   ip: "::"
   ##   module: ejabberd_xmlrpc
   ##   access_commands: {}
-  - 
-    port: 5280
-    module: ejabberd_http
-    request_handlers:
-      "/websocket": ejabberd_http_ws
-      "/api": mod_http_api
-    ##  "/pub/archive": mod_http_fileserver
-    web_admin: true
-    http_bind: true
-    ## register: true
-    captcha: true
+
+  ##
+  ## To enable secure http upload
+  ##
+  ## - 
+  ##   port: 5444
+  ##   ip: "::"
+  ##   module: ejabberd_http
+  ##   request_handlers:
+  ##     "": mod_http_upload
+  ##   tls: true
+  ##   certfile: 'CERTFILE'
+  ##   protocol_options: 'TLSOPTS'
+  ##   dhfile: 'DHFILE'
+  ##   ciphers: 'CIPHERS'
+
+## Disabling digest-md5 SASL authentication. digest-md5 requires plain-text
+## password storage (see auth_password_format option).
+## disable_sasl_mechanisms: "digest-md5"
 
 ###.  ==================
 ###'  S2S GLOBAL OPTIONS
@@ -197,18 +234,16 @@ listen:
 ## Allowed values are: false optional required required_trusted
 ## You must specify a certificate file.
 ##
-## s2s_use_starttls: optional
+## s2s_use_starttls: required
 
 ##
 ## s2s_certfile: Specify a certificate file.
 ##
-## s2s_certfile: "/path/to/ssl.pem"
+## s2s_certfile: 'CERTFILE'
 
 ## Custom OpenSSL options
 ##
-## s2s_protocol_options:
-##   - "no_sslv3"
-##   - "no_tlsv1"
+## s2s_protocol_options: 'TLSOPTS'
 
 ##
 ## domain_certfile: Specify a different certificate for each served hostname.
@@ -630,7 +665,7 @@ language: "en"
 ##
 modules: 
   mod_adhoc: {}
-  ## mod_admin_extra: {}
+  mod_admin_extra: {}
   mod_announce: # recommends mod_adhoc
     access: announce
   mod_blocking: {} # requires mod_privacy
@@ -638,15 +673,25 @@ modules:
   mod_carboncopy: {}
   mod_client_state: {}
   mod_configure: {} # requires mod_adhoc
-  ##mod_delegation: {} # for xep0356
+  ## mod_delegation: {} # for xep0356
   mod_disco: {}
-  ## mod_echo: {}
+  mod_echo: {}
   mod_irc: {}
   mod_bosh: {}
   ## mod_http_fileserver:
   ##   docroot: "/var/www"
   ##   accesslog: "/var/log/ejabberd/access.log"
+  ## mod_http_upload:
+  ##   # docroot: "@HOME@/upload"
+  ##   put_url: "https://@HOST@:5444"
+  ##   thumbnail: false # otherwise needs the identify command from ImageMagick installed
+  ## mod_http_upload_quota:
+  ##   max_days: 30
   mod_last: {}
+  ## XEP-0313: Message Archive Management
+  ## You might want to setup a SQL backend for MAM because the mnesia database is
+  ## limited to 2GB which might be exceeded on large servers
+  ## mod_mam: {} # for xep0313, mnesia is limited to 2GB, better use an SQL backend
   mod_muc: 
     ## host: "conference.@HOST@"
     access:
@@ -655,6 +700,7 @@ modules:
       - allow: admin
     access_create: muc_create
     access_persistent: muc_create
+  mod_muc_admin: {}
   ## mod_muc_log: {}
   ## mod_multicast: {}
   mod_offline: 

include/mod_muc_room.hrl

@@ -28,9 +28,8 @@
 
 -record(lqueue,
 {
-    queue = queue:new() :: ?TQUEUE,
-    len = 0             :: integer(),
-    max = 0             :: integer()
+    queue   :: p1_queue:queue(),
+    max = 0 :: integer()
 }).
 
 -type lqueue() :: #lqueue{}.
@@ -112,11 +111,11 @@
     robots                  = (?DICT):new() :: ?TDICT,
     nicks                   = (?DICT):new() :: ?TDICT,
     affiliations            = (?DICT):new() :: ?TDICT,
-    history                 = #lqueue{} :: lqueue(),
+    history                 :: lqueue(),
     subject                 = <<"">> :: binary(),
     subject_author          = <<"">> :: binary(),
     just_created            = false :: boolean(),
     activity                = treap:empty() :: treap:treap(),
     room_shaper             = none :: shaper:shaper(),
-    room_queue              = queue:new() :: ?TQUEUE
+    room_queue              :: p1_queue:queue() | undefined
 }).

include/pubsub.hrl

@@ -145,6 +145,7 @@
 -record(pubsub_state,
 {
     stateid               ,% :: {jlib:ljid(), mod_pubsub:nodeIdx()},
+    nodeidx               ,% :: mod_pubsub:nodeIdx(),
     items         = []    ,% :: [mod_pubsub:itemId(),...],
     affiliation   = 'none',% :: mod_pubsub:affiliation(),
     subscriptions = []    % :: [{mod_pubsub:subscription(), mod_pubsub:subId()}]
@@ -153,6 +154,7 @@
 -record(pubsub_item,
 {
     itemid                           ,% :: {mod_pubsub:itemId(), mod_pubsub:nodeIdx()},
+    nodeidx                          ,% :: mod_pubsub:nodeIdx(),
     creation     = {unknown, unknown},% :: {erlang:timestamp(), jlib:ljid()},
     modification = {unknown, unknown},% :: {erlang:timestamp(), jlib:ljid()},
     payload      = []                % :: mod_pubsub:payload()

mix.exs

@@ -34,7 +34,7 @@ defmodule Ejabberd.Mixfile do
 
   defp erlc_options do
     # Use our own includes + includes from all dependencies
-    includes = ["include"] ++ deps_include(["fast_xml", "xmpp"])
+    includes = ["include"] ++ deps_include(["fast_xml", "xmpp", "p1_utils"])
     [:debug_info, {:d, :ELIXIR_ENABLED}] ++ Enum.map(includes, fn(path) -> {:i, path} end)
   end
 

rebar.config

@@ -19,7 +19,7 @@
 %%%----------------------------------------------------------------------
 
 {deps, [{lager, ".*", {git, "https://github.com/basho/lager", {tag, "3.2.1"}}},
-        {p1_utils, ".*", {git, "https://github.com/processone/p1_utils", {tag, "1.0.7"}}},
+        {p1_utils, ".*", {git, "https://github.com/processone/p1_utils", {tag, "1.0.8"}}},
         {cache_tab, ".*", {git, "https://github.com/processone/cache_tab", {tag, "1.0.7"}}},
         {fast_tls, ".*", {git, "https://github.com/processone/fast_tls", {tag, "1.0.11"}}},
         {stringprep, ".*", {git, "https://github.com/processone/stringprep", {tag, "1.0.8"}}},
@@ -81,6 +81,7 @@
             {i, "include"},
 	    {i, "deps/fast_xml/include"},
 	    {i, "deps/xmpp/include"},
+	    {i, "deps/p1_utils/include"},
             {if_var_false, debug, no_debug_info},
             {if_var_true, debug, debug_info},
             {if_var_true, roster_gateway_workaround, {d, 'ROSTER_GATWAY_WORKAROUND'}},
@@ -134,6 +135,7 @@
 
 {eunit_compile_opts, [{i, "tools"},
 		      {i, "include"},
+		      {i, "deps/p1_utils/include"},
 		      {i, "deps/fast_xml/include"},
 		      {i, "deps/xmpp/include"}]}.
 

sql/lite.sql

@@ -275,6 +275,27 @@ CREATE TABLE muc_registered (
 CREATE INDEX i_muc_registered_nick ON muc_registered (nick);
 CREATE UNIQUE INDEX i_muc_registered_jid_host ON muc_registered (jid, host);
 
+CREATE TABLE muc_online_room (
+    name text NOT NULL,
+    host text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_muc_online_room_name_host ON muc_online_room (name, host);
+
+CREATE TABLE muc_online_users (
+    username text NOT NULL,
+    server text NOT NULL,
+    resource text NOT NULL,
+    name text NOT NULL,
+    host text NOT NULL,
+    node text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_muc_online_users ON muc_online_users (username, server, resource, name, host);
+CREATE INDEX i_muc_online_users_us ON muc_online_users (username, server);
+
 CREATE TABLE irc_custom (
     jid text NOT NULL,
     host text NOT NULL,
@@ -319,3 +340,44 @@ CREATE TABLE oauth_token (
     scope text NOT NULL,
     expire bigint NOT NULL
 );
+
+CREATE TABLE route (
+    domain text NOT NULL,
+    server_host text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL,
+    local_hint text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_route ON route(domain, server_host, node, pid);
+CREATE INDEX i_route_domain ON route(domain);
+
+CREATE TABLE bosh (
+    sid text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_bosh_sid ON bosh(sid);
+
+CREATE TABLE carboncopy (
+    username text NOT NULL,
+    resource text NOT NULL,
+    namespace text NOT NULL,
+    node text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_carboncopy_ur ON carboncopy (username, resource);
+CREATE INDEX i_carboncopy_user ON carboncopy (username);
+
+CREATE TABLE proxy65 (
+    sid text NOT NULL,
+    pid_t text NOT NULL,
+    pid_i text NOT NULL,
+    node_t text NOT NULL,
+    node_i text NOT NULL,
+    jid_i text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_proxy65_sid ON proxy65 (sid);
+CREATE INDEX i_proxy65_jid ON proxy65 (jid_i);

sql/mssql.sql

@@ -125,6 +125,30 @@ CREATE TABLE [dbo].[muc_room] (
 CREATE UNIQUE CLUSTERED INDEX [muc_room_name_host] ON [muc_room] (name, host)
 WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
 
+CREATE TABLE [dbo].[muc_online_room] (
+        [name] [varchar] (250) NOT NULL,
+	[host] [varchar] (250) NOT NULL,
+	[node] [text] NOT NULL,
+	[pid] [text] NOT NULL
+);
+
+CREATE UNIQUE CLUSTERED INDEX [muc_online_room_name_host] ON [muc_online_room] (name, host)
+WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
+
+CREATE TABLE [dbo].[muc_online_users] (
+    [username] [varchar] (250) NOT NULL,
+    [server] [varchar] (250) NOT NULL,
+    [resource] [varchar] (250) NOT NULL,
+    [name] [varchar] (250) NOT NULL,
+    [host] [varchar] (250) NOT NULL,
+    node text NOT NULL
+);
+
+CREATE UNIQUE CLUSTERED INDEX [muc_online_users_i] ON [muc_online_users] (username, server, resource, name, host)
+WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
+CREATE UNIQUE CLUSTERED INDEX [muc_online_users_us] ON [muc_online_users] (username, server);
+WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
+
 CREATE TABLE [dbo].[privacy_default_list] (
         [username] [varchar] (250) NOT NULL,
         [name] [varchar] (250) NOT NULL,
@@ -490,3 +514,40 @@ CREATE TABLE [dbo].[oauth_token] (
         [token] ASC
 )WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON)
 ) TEXTIMAGE_ON [PRIMARY];
+
+CREATE TABLE [dbo].[route] (
+    [domain] [varchar] (255) NOT NULL,
+    [server_host] [varchar] (255) NOT NULL,
+    [node] [varchar] (255) NOT NULL,
+    [pid] [varchar](100) NOT NULL,
+    [local_hint] [text] NOT NULL
+);
+
+CREATE UNIQUE CLUSTERED INDEX [route_i] ON [route] (domain, server_host, node, pid)
+WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
+
+CREATE INDEX [route_domain] ON [route] (domain)
+WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
+
+CREATE TABLE [dbo].[bosh] (
+    [sid] [varchar] (255) NOT NULL,
+    [node] [varchar] (255) NOT NULL,
+    [pid] [varchar](100) NOT NULL
+ CONSTRAINT [bosh_PRIMARY] PRIMARY KEY CLUSTERED 
+(
+        [sid] ASC
+)WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, IGNORE_DUP_KEY = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON)
+) TEXTIMAGE_ON [PRIMARY];
+
+CREATE TABLE [dbo].[carboncopy] (
+    [username] [varchar] (255) NOT NULL,
+    [resource] [varchar] (255) NOT NULL,
+    [namespace] [varchar] (255) NOT NULL,
+    [node] [varchar] (255) NOT NULL
+);
+
+CREATE UNIQUE CLUSTERED INDEX [carboncopy_ur] ON [carboncopy] (username, resource)
+WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);
+
+CREATE INDEX [carboncopy_user] ON [carboncopy] (username)
+WITH (PAD_INDEX = OFF, STATISTICS_NORECOMPUTE = OFF, ALLOW_ROW_LOCKS = ON, ALLOW_PAGE_LOCKS = ON);

sql/mysql.sql

@@ -291,6 +291,27 @@ CREATE TABLE muc_registered (
 CREATE INDEX i_muc_registered_nick USING BTREE ON muc_registered(nick(75));
 CREATE UNIQUE INDEX i_muc_registered_jid_host USING BTREE ON muc_registered(jid(75), host(75));
 
+CREATE TABLE muc_online_room (
+    name text NOT NULL,
+    host text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL
+) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+CREATE UNIQUE INDEX i_muc_online_room_name_host USING BTREE ON muc_online_room(name(75), host(75));
+
+CREATE TABLE muc_online_users (
+    username text NOT NULL,
+    server text NOT NULL,
+    resource text NOT NULL,
+    name text NOT NULL,
+    host text NOT NULL,
+    node text NOT NULL
+) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+CREATE UNIQUE INDEX i_muc_online_users USING BTREE ON muc_online_users(username(75), server(75), resource(75), name(75), host(75));
+CREATE INDEX i_muc_online_users_us USING BTREE ON muc_online_users(username(75), server(75));
+
 CREATE TABLE irc_custom (
     jid text NOT NULL,
     host text NOT NULL,
@@ -335,3 +356,44 @@ CREATE TABLE oauth_token (
     scope text NOT NULL,
     expire bigint NOT NULL
 ) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+CREATE TABLE route (
+    domain text NOT NULL,
+    server_host text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL,
+    local_hint text NOT NULL
+) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+CREATE UNIQUE INDEX i_route ON route(domain(75), server_host(75), node(75), pid(75));
+CREATE INDEX i_route_domain ON route(domain(75));
+
+CREATE TABLE bosh (
+    sid text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL
+) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+CREATE UNIQUE INDEX i_bosh_sid ON bosh(sid(75));
+
+CREATE TABLE carboncopy (
+    username text NOT NULL,
+    resource text NOT NULL,
+    namespace text NOT NULL,
+    node text NOT NULL
+) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+CREATE UNIQUE INDEX i_carboncopy_ur ON carboncopy (username(75), resource(75));
+CREATE INDEX i_carboncopy_user ON carboncopy (username(75));
+
+CREATE TABLE proxy65 (
+    sid text NOT NULL,
+    pid_t text NOT NULL,
+    pid_i text NOT NULL,
+    node_t text NOT NULL,
+    node_i text NOT NULL,
+    jid_i text NOT NULL
+) ENGINE=InnoDB CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+
+CREATE UNIQUE INDEX i_proxy65_sid ON proxy65 (sid(191));
+CREATE INDEX i_proxy65_jid ON proxy65 (jid_i(191));

sql/pg.sql

@@ -293,6 +293,27 @@ CREATE TABLE muc_registered (
 CREATE INDEX i_muc_registered_nick ON muc_registered USING btree (nick);
 CREATE UNIQUE INDEX i_muc_registered_jid_host ON muc_registered USING btree (jid, host);
 
+CREATE TABLE muc_online_room (
+    name text NOT NULL,
+    host text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_muc_online_room_name_host ON muc_online_room USING btree (name, host);
+
+CREATE TABLE muc_online_users (
+    username text NOT NULL,
+    server text NOT NULL,
+    resource text NOT NULL,
+    name text NOT NULL,
+    host text NOT NULL,
+    node text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_muc_online_users ON muc_online_users USING btree (username, server, resource, name, host);
+CREATE INDEX i_muc_online_users_us ON muc_online_users USING btree (username, server);
+
 CREATE TABLE irc_custom (
     jid text NOT NULL,
     host text NOT NULL,
@@ -339,3 +360,44 @@ CREATE TABLE oauth_token (
 );
 
 CREATE UNIQUE INDEX i_oauth_token_token ON oauth_token USING btree (token);
+
+CREATE TABLE route (
+    domain text NOT NULL,
+    server_host text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL,
+    local_hint text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_route ON route USING btree (domain, server_host, node, pid);
+CREATE INDEX i_route_domain ON route USING btree (domain);
+
+CREATE TABLE bosh (
+    sid text NOT NULL,
+    node text NOT NULL,
+    pid text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_bosh_sid ON bosh USING btree (sid);
+
+CREATE TABLE carboncopy (
+    username text NOT NULL,
+    resource text NOT NULL,
+    namespace text NOT NULL,
+    node text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_carboncopy_ur ON carboncopy USING btree (username, resource);
+CREATE INDEX i_carboncopy_user ON carboncopy USING btree (username);
+
+CREATE TABLE proxy65 (
+    sid text NOT NULL,
+    pid_t text NOT NULL,
+    pid_i text NOT NULL,
+    node_t text NOT NULL,
+    node_i text NOT NULL,
+    jid_i text NOT NULL
+);
+
+CREATE UNIQUE INDEX i_proxy65_sid ON proxy65 USING btree (sid);
+CREATE INDEX i_proxy65_jid ON proxy65 USING btree (jid_i);

src/cyrsasl_scram.erl

@@ -128,14 +128,14 @@ mech_step(#state{step = 2} = State, ClientIn) ->
 				      str:substr(ClientIn,
                                                  str:str(ClientIn, <<"n=">>)),
 				  ServerNonce =
-				      jlib:encode_base64(randoms:bytes(?NONCE_LENGTH)),
+				      misc:encode_base64(randoms:bytes(?NONCE_LENGTH)),
 				  ServerFirstMessage =
                                         iolist_to_binary(
                                           ["r=",
                                            ClientNonce,
                                            ServerNonce,
                                            ",", "s=",
-                                           jlib:encode_base64(Salt),
+                                           misc:encode_base64(Salt),
                                            ",", "i=",
                                            integer_to_list(IterationCount)]),
 				  {continue, ServerFirstMessage,
@@ -161,7 +161,7 @@ mech_step(#state{step = 4} = State, ClientIn) ->
        ClientProofAttribute] ->
 	  case parse_attribute(GS2ChannelBindingAttribute) of
 	    {$c, CVal} ->
-		ChannelBindingSupport = binary:at(jlib:decode_base64(CVal), 0),
+		ChannelBindingSupport = binary:at(misc:decode_base64(CVal), 0),
 		if (ChannelBindingSupport == $n)
 		  or (ChannelBindingSupport == $y) ->
 		    Nonce = <<(State#state.client_nonce)/binary,
@@ -170,7 +170,7 @@ mech_step(#state{step = 4} = State, ClientIn) ->
 			{$r, CompareNonce} when CompareNonce == Nonce ->
 			    case parse_attribute(ClientProofAttribute) of
 			    {$p, ClientProofB64} ->
-				  ClientProof = jlib:decode_base64(ClientProofB64),
+				  ClientProof = misc:decode_base64(ClientProofB64),
 				  AuthMessage = iolist_to_binary(
 						    [State#state.auth_message,
 						     ",",
@@ -191,7 +191,7 @@ mech_step(#state{step = 4} = State, ClientIn) ->
 					       {auth_module, State#state.auth_module},
 					       {authzid, State#state.username}],
 					  <<"v=",
-					    (jlib:encode_base64(ServerSignature))/binary>>};
+					    (misc:encode_base64(ServerSignature))/binary>>};
 				     true -> {error, not_authorized, State#state.username}
 				  end;
 			    _ -> {error, bad_attribute}

src/ejabberd_app.erl

@@ -50,6 +50,7 @@ start(normal, _Args) ->
     setup_if_elixir_conf_used(),
     ejabberd_config:start(),
     set_settings_from_config(),
+    file_queue_init(),
     maybe_add_nameservers(),
     connect_nodes(),
     case ejabberd_sup:start_link() of
@@ -169,6 +170,16 @@ set_settings_from_config() ->
                  60),
     net_kernel:set_net_ticktime(Ticktime).
 
+file_queue_init() ->
+    QueueDir = case ejabberd_config:queue_dir() of
+		   undefined ->
+		       MnesiaDir = mnesia:system_info(directory),
+		       filename:join(MnesiaDir, "queue");
+		   Path ->
+		       Path
+	       end,
+    p1_queue:start(QueueDir).
+
 start_apps() ->
     crypto:start(),
     ejabberd:start_app(sasl),

src/ejabberd_auth.erl

@@ -510,8 +510,8 @@ auth_modules(Server) ->
     Default = ejabberd_config:default_db(LServer, ?MODULE),
     Methods = ejabberd_config:get_option(
                 {auth_method, LServer}, opt_type(auth_method), [Default]),
-    [jlib:binary_to_atom(<<"ejabberd_auth_",
-			   (jlib:atom_to_binary(M))/binary>>)
+    [misc:binary_to_atom(<<"ejabberd_auth_",
+			   (misc:atom_to_binary(M))/binary>>)
      || M <- Methods].
 
 export(Server) ->

src/ejabberd_auth_ldap.erl

@@ -361,8 +361,8 @@ result_attrs(#state{uids = UIDs,
 %%%----------------------------------------------------------------------
 parse_options(Host) ->
     Cfg = eldap_utils:get_config(Host, []),
-    Eldap_ID = jlib:atom_to_binary(gen_mod:get_module_proc(Host, ?MODULE)),
-    Bind_Eldap_ID = jlib:atom_to_binary(
+    Eldap_ID = misc:atom_to_binary(gen_mod:get_module_proc(Host, ?MODULE)),
+    Bind_Eldap_ID = misc:atom_to_binary(
                       gen_mod:get_module_proc(Host, bind_ejabberd_auth_ldap)),
     UIDsTemp = gen_mod:get_opt(
                  {ldap_uids, Host}, [],

src/ejabberd_auth_mnesia.erl

@@ -129,7 +129,7 @@ check_password(User, AuthzId, Server, Password, Digest,
 		     true -> (Passwd == Password) and (Password /= <<"">>)
 		  end;
 	      [#passwd{password = Scram}] when is_record(Scram, scram) ->
-		  Passwd = jlib:decode_base64(Scram#scram.storedkey),
+		  Passwd = misc:decode_base64(Scram#scram.storedkey),
 		  DigRes = if Digest /= <<"">> ->
 				   Digest == DigestGen(Passwd);
 			      true -> false
@@ -294,9 +294,9 @@ get_password(User, Server) ->
 	  Password;
       [#passwd{password = Scram}]
 	  when is_record(Scram, scram) ->
-	  {jlib:decode_base64(Scram#scram.storedkey),
-	   jlib:decode_base64(Scram#scram.serverkey),
-	   jlib:decode_base64(Scram#scram.salt),
+	  {misc:decode_base64(Scram#scram.storedkey),
+	   misc:decode_base64(Scram#scram.serverkey),
+	   misc:decode_base64(Scram#scram.salt),
 	   Scram#scram.iterationcount};
       _ -> false
     end.
@@ -480,9 +480,9 @@ password_to_scram(Password, IterationCount) ->
     StoredKey =
 	scram:stored_key(scram:client_key(SaltedPassword)),
     ServerKey = scram:server_key(SaltedPassword),
-    #scram{storedkey = jlib:encode_base64(StoredKey),
-	   serverkey = jlib:encode_base64(ServerKey),
-	   salt = jlib:encode_base64(Salt),
+    #scram{storedkey = misc:encode_base64(StoredKey),
+	   serverkey = misc:encode_base64(ServerKey),
+	   salt = misc:encode_base64(Salt),
 	   iterationcount = IterationCount}.
 
 is_password_scram_valid(Password, Scram) ->
@@ -491,12 +491,12 @@ is_password_scram_valid(Password, Scram) ->
 	    false;
 	_ ->
 	    IterationCount = Scram#scram.iterationcount,
-	    Salt = jlib:decode_base64(Scram#scram.salt),
+	    Salt = misc:decode_base64(Scram#scram.salt),
 	    SaltedPassword = scram:salted_password(Password, Salt,
 						   IterationCount),
 	    StoredKey =
 		scram:stored_key(scram:client_key(SaltedPassword)),
-	    jlib:decode_base64(Scram#scram.storedkey) == StoredKey
+	    misc:decode_base64(Scram#scram.storedkey) == StoredKey
     end.
 
 export(_Server) ->

src/ejabberd_auth_riak.erl

@@ -108,7 +108,7 @@ check_password(User, AuthzId, Server, Password, Digest,
 	  end;
       {ok, #passwd{password = Scram}}
 	  when is_record(Scram, scram) ->
-	  Passwd = jlib:decode_base64(Scram#scram.storedkey),
+	  Passwd = misc:decode_base64(Scram#scram.storedkey),
 	  DigRes = if Digest /= <<"">> ->
 			  Digest == DigestGen(Passwd);
 		      true -> false
@@ -213,9 +213,9 @@ get_password(User, Server) ->
 	  Password;
       {ok, #passwd{password = Scram}}
 	  when is_record(Scram, scram) ->
-	  {jlib:decode_base64(Scram#scram.storedkey),
-	   jlib:decode_base64(Scram#scram.serverkey),
-	   jlib:decode_base64(Scram#scram.salt),
+	  {misc:decode_base64(Scram#scram.storedkey),
+	   misc:decode_base64(Scram#scram.serverkey),
+	   misc:decode_base64(Scram#scram.salt),
 	   Scram#scram.iterationcount};
       _ -> false
     end.
@@ -287,9 +287,9 @@ password_to_scram(Password, IterationCount) ->
     StoredKey =
 	scram:stored_key(scram:client_key(SaltedPassword)),
     ServerKey = scram:server_key(SaltedPassword),
-    #scram{storedkey = jlib:encode_base64(StoredKey),
-	   serverkey = jlib:encode_base64(ServerKey),
-	   salt = jlib:encode_base64(Salt),
+    #scram{storedkey = misc:encode_base64(StoredKey),
+	   serverkey = misc:encode_base64(ServerKey),
+	   salt = misc:encode_base64(Salt),
 	   iterationcount = IterationCount}.
 
 is_password_scram_valid(Password, Scram) ->
@@ -298,12 +298,12 @@ is_password_scram_valid(Password, Scram) ->
 	    false;
 	_ ->
 	    IterationCount = Scram#scram.iterationcount,
-	    Salt = jlib:decode_base64(Scram#scram.salt),
+	    Salt = misc:decode_base64(Scram#scram.salt),
 	    SaltedPassword = scram:salted_password(Password, Salt,
 						   IterationCount),
 	    StoredKey =
 		scram:stored_key(scram:client_key(SaltedPassword)),
-	    jlib:decode_base64(Scram#scram.storedkey) == StoredKey
+	    misc:decode_base64(Scram#scram.storedkey) == StoredKey
     end.
 
 export(_Server) ->

src/ejabberd_auth_sql.erl

@@ -301,9 +301,9 @@ get_password(User, Server) ->
                                  LServer, LUser) of
                         {selected,
                          [{StoredKey, ServerKey, Salt, IterationCount}]} ->
-                            {jlib:decode_base64(StoredKey),
-                             jlib:decode_base64(ServerKey),
-                             jlib:decode_base64(Salt),
+                            {misc:decode_base64(StoredKey),
+                             misc:decode_base64(ServerKey),
+                             misc:decode_base64(Salt),
                              IterationCount};
                         _ -> false
                     end;
@@ -423,9 +423,9 @@ password_to_scram(Password, IterationCount) ->
     StoredKey =
 	scram:stored_key(scram:client_key(SaltedPassword)),
     ServerKey = scram:server_key(SaltedPassword),
-    #scram{storedkey = jlib:encode_base64(StoredKey),
-	   serverkey = jlib:encode_base64(ServerKey),
-	   salt = jlib:encode_base64(Salt),
+    #scram{storedkey = misc:encode_base64(StoredKey),
+	   serverkey = misc:encode_base64(ServerKey),
+	   salt = misc:encode_base64(Salt),
 	   iterationcount = IterationCount}.
 
 is_password_scram_valid_stored(Pass, {scram,Pass,<<>>,<<>>,0}, LUser, LServer) ->
@@ -443,12 +443,12 @@ is_password_scram_valid(Password, Scram) ->
 	    false;
 	_ ->
 	    IterationCount = Scram#scram.iterationcount,
-	    Salt = jlib:decode_base64(Scram#scram.salt),
+	    Salt = misc:decode_base64(Scram#scram.salt),
 	    SaltedPassword = scram:salted_password(Password, Salt,
 						   IterationCount),
 	    StoredKey =
 		scram:stored_key(scram:client_key(SaltedPassword)),
-	    jlib:decode_base64(Scram#scram.storedkey) == StoredKey
+	    misc:decode_base64(Scram#scram.storedkey) == StoredKey
     end.
 
 -define(BATCH_SIZE, 1000).

src/ejabberd_bosh.erl

@@ -96,8 +96,8 @@
 -record(state,
 	{host = <<"">>                            :: binary(),
          sid = <<"">>                             :: binary(),
-         el_ibuf = buf_new()                      :: ?TQUEUE,
-         el_obuf = buf_new()                      :: ?TQUEUE,
+         el_ibuf                                  :: p1_queue:queue(),
+         el_obuf                                  :: p1_queue:queue(),
          shaper_state = none                      :: shaper:shaper(),
          c2s_pid                                  :: pid() | undefined,
 	 xmpp_ver = <<"">>                        :: binary(),
@@ -111,7 +111,7 @@
          max_concat = unlimited                   :: unlimited | non_neg_integer(),
 	 responses = gb_trees:empty()             :: ?TGB_TREE,
 	 receivers = gb_trees:empty()             :: ?TGB_TREE,
-	 shaped_receivers = queue:new()           :: ?TQUEUE,
+	 shaped_receivers                         :: p1_queue:queue(),
          ip                                       :: inet:ip_address(),
          max_requests = 1                         :: non_neg_integer()}).
 
@@ -305,10 +305,10 @@ init([#body{attrs = Attrs}, IP, SID]) ->
                            false) of
                         true ->
                             JID = make_random_jid(XMPPDomain),
-                            {buf_new(), [{jid, JID} | Opts2]};
+                            {buf_new(XMPPDomain), [{jid, JID} | Opts2]};
                         false ->
                             {buf_in([make_xmlstreamstart(XMPPDomain, XMPPVer)],
-                                    buf_new()),
+                                    buf_new(XMPPDomain)),
                              Opts2}
 		    end,
     ejabberd_socket:start(ejabberd_c2s, ?MODULE, Socket,
@@ -321,10 +321,12 @@ init([#body{attrs = Attrs}, IP, SID]) ->
                                        fun(unlimited) -> unlimited;
                                           (N) when is_integer(N), N>0 -> N
                                        end, unlimited),
+    ShapedReceivers = buf_new(XMPPDomain, ?MAX_SHAPED_REQUESTS_QUEUE_LEN),
     State = #state{host = XMPPDomain, sid = SID, ip = IP,
 		   xmpp_ver = XMPPVer, el_ibuf = InBuf,
-		   max_concat = MaxConcat, el_obuf = buf_new(),
+		   max_concat = MaxConcat, el_obuf = buf_new(XMPPDomain),
 		   inactivity_timeout = Inactivity,
+		   shaped_receivers = ShapedReceivers,
 		   shaper_state = ShaperState},
     NewState = restart_inactivity_timer(State),
     mod_bosh:open_session(SID, self()),
@@ -417,15 +419,15 @@ active(#body{attrs = Attrs, size = Size} = Req, From,
 	shaper:update(State#state.shaper_state, Size),
     State1 = State#state{shaper_state = ShaperState},
     if Pause > 0 ->
-	   QLen = queue:len(State1#state.shaped_receivers),
-	   if QLen < (?MAX_SHAPED_REQUESTS_QUEUE_LEN) ->
-		  TRef = start_shaper_timer(Pause),
-		  Q = queue:in({TRef, From, Req},
-			       State1#state.shaped_receivers),
-		  State2 = stop_inactivity_timer(State1),
-		  {next_state, active,
-		   State2#state{shaped_receivers = Q}};
-	      true ->
+	    TRef = start_shaper_timer(Pause),
+	    try p1_queue:in({TRef, From, Req},
+			    State1#state.shaped_receivers) of
+		Q ->
+		    State2 = stop_inactivity_timer(State1),
+		    {next_state, active,
+		     State2#state{shaped_receivers = Q}}
+	    catch error:full ->
+		  cancel_timer(TRef),
 		  RID = get_attr(rid, Attrs),
 		  reply_stop(State1,
 			     #body{http_reason = <<"Too many requests">>,
@@ -572,7 +574,7 @@ handle_sync_event({send_xml, El}, _From, StateName,
 	   reply(State2, Body#body{els = Els},
 		 State2#state.prev_rid, From)};
       none ->
-	  State2 = case queue:out(State1#state.shaped_receivers)
+	  State2 = case p1_queue:out(State1#state.shaped_receivers)
 		       of
 		     {{value, {TRef, From, Body}}, Q} ->
 			 cancel_timer(TRef),
@@ -601,7 +603,7 @@ handle_info({timeout, TRef, inactive}, _StateName,
     {stop, normal, State};
 handle_info({timeout, TRef, shaper_timeout}, StateName,
 	    State) ->
-    case queue:out(State#state.shaped_receivers) of
+    case p1_queue:out(State#state.shaped_receivers) of
       {{value, {TRef, From, Req}}, Q} ->
 	  (?GEN_FSM):send_event(self(), {Req, From}),
 	  {next_state, StateName,
@@ -646,9 +648,13 @@ code_change(_OldVsn, StateName, State, _Extra) ->
 
 print_state(State) -> State.
 
-route_els(#state{el_ibuf = Buf} = State) ->
-    route_els(State#state{el_ibuf = buf_new()},
-	      buf_to_list(Buf)).
+route_els(#state{el_ibuf = Buf, c2s_pid = C2SPid} = State) ->
+    NewBuf = p1_queue:dropwhile(
+	       fun(El) ->
+		       ?GEN_FSM:send_event(C2SPid, El),
+		       true
+	       end, Buf),
+    State#state{el_ibuf = NewBuf}.
 
 route_els(State, Els) ->
     case State#state.c2s_pid of
@@ -734,7 +740,7 @@ bounce_receivers(State, Reason) ->
 					RID = get_attr(rid, Attrs),
 					{RID, {From, Body}}
 				end,
-				queue:to_list(State#state.shaped_receivers)),
+				p1_queue:to_list(State#state.shaped_receivers)),
     lists:foldl(fun ({RID, {From, Body}}, AccState) ->
 			NewBody = if Reason == closed ->
 					 #body{http_reason =
@@ -752,7 +758,7 @@ bounce_receivers(State, Reason) ->
 		State, Receivers ++ ShapedReceivers).
 
 bounce_els_from_obuf(State) ->
-    lists:foreach(
+    p1_queue:foreach(
       fun({xmlstreamelement, El}) ->
 	      try xmpp:decode(El, ?NS_CLIENT, [ignore_els]) of
 		  Pkt when ?is_stanza(Pkt) ->
@@ -769,7 +775,7 @@ bounce_els_from_obuf(State) ->
 	      end;
 	 (_) ->
 	      ok
-      end, buf_to_list(State#state.el_obuf)).
+      end, State#state.el_obuf).
 
 is_valid_key(<<"">>, <<"">>) -> true;
 is_valid_key(PrevKey, Key) ->
@@ -1029,26 +1035,33 @@ get_attr(Attr, Attrs, Default) ->
       _ -> Default
     end.
 
-buf_new() -> queue:new().
+buf_new(Host) ->
+    buf_new(Host, unlimited).
+
+buf_new(Host, Limit) ->
+    QueueType = case gen_mod:get_module_opt(
+		       Host, mod_bosh, queue_type,
+		       mod_bosh:mod_opt_type(queue_type)) of
+		    undefined -> ejabberd_config:default_queue_type(Host);
+		    T -> T
+		end,
+    p1_queue:new(QueueType, Limit).
 
 buf_in(Xs, Buf) ->
-    lists:foldl(fun (X, Acc) -> queue:in(X, Acc) end, Buf,
-		Xs).
+    lists:foldl(fun p1_queue:in/2, Buf, Xs).
 
 buf_out(Buf, Num) when is_integer(Num), Num > 0 ->
     buf_out(Buf, Num, []);
-buf_out(Buf, _) -> {queue:to_list(Buf), buf_new()}.
+buf_out(Buf, _) -> {p1_queue:to_list(Buf), p1_queue:clear(Buf)}.
 
 buf_out(Buf, 0, Els) -> {lists:reverse(Els), Buf};
 buf_out(Buf, I, Els) ->
-    case queue:out(Buf) of
+    case p1_queue:out(Buf) of
       {{value, El}, NewBuf} ->
 	  buf_out(NewBuf, I - 1, [El | Els]);
       {empty, _} -> buf_out(Buf, 0, Els)
     end.
 
-buf_to_list(Buf) -> queue:to_list(Buf).
-
 cancel_timer(TRef) when is_reference(TRef) ->
     (?GEN_FSM):cancel_timer(TRef);
 cancel_timer(_) -> false.

src/ejabberd_c2s.erl

@@ -438,7 +438,7 @@ handle_auth_success(User, Mech, AuthModule,
     ?INFO_MSG("(~s) Accepted c2s ~s authentication for ~s@~s by ~s backend from ~s",
 	      [SockMod:pp(Socket), Mech, User, LServer,
 	       ejabberd_auth:backend_type(AuthModule),
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP))]),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP))]),
     State1 = State#{auth_module => AuthModule},
     ejabberd_hooks:run_fold(c2s_auth_result, LServer, State1, [true, User]).
 
@@ -450,7 +450,7 @@ handle_auth_failure(User, Mech, Reason,
 	       if User /= <<"">> -> ["for ", User, "@", LServer, " "];
 		  true -> ""
 	       end,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP)), Reason]),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP)), Reason]),
     ejabberd_hooks:run_fold(c2s_auth_result, LServer, State, [false, User]).
 
 handle_unbinded_packet(Pkt, #{lserver := LServer} = State) ->

src/ejabberd_config.erl

@@ -37,7 +37,7 @@
 	 env_binary_to_list/2, opt_type/1, may_hide_data/1,
 	 is_elixir_enabled/0, v_dbs/1, v_dbs_mods/1,
 	 default_db/1, default_db/2, default_ram_db/1, default_ram_db/2,
-	 fsm_limit_opts/1]).
+	 default_queue_type/1, queue_dir/0, fsm_limit_opts/1]).
 
 -export([start/2]).
 
@@ -73,7 +73,7 @@ start() ->
                     nocookie ->
                         str:sha(randoms:get_string());
                     Cookie ->
-                        str:sha(jlib:atom_to_binary(Cookie))
+                        str:sha(misc:atom_to_binary(Cookie))
                 end,
     State2 = set_option({node_start, global}, UnixTime, State1),
     State3 = set_option({shared_key, global}, SharedKey, State2),
@@ -894,7 +894,8 @@ has_option(Opt) ->
     get_option(Opt, fun(_) -> true end, false).
 
 init_module_db_table(Modules) ->
-    catch ets:new(module_db, [named_table, public, bag]),
+    catch ets:new(module_db, [named_table, public, bag,
+			      {read_concurrency, true}]),
     %% Dirty hack for mod_pubsub
     ets:insert(module_db, {mod_pubsub, mnesia}),
     ets:insert(module_db, {mod_pubsub, sql}),
@@ -1455,9 +1456,13 @@ opt_type(default_ram_db) ->
     fun(T) when is_atom(T) -> T end;
 opt_type(loglevel) ->
     fun (P) when P >= 0, P =< 5 -> P end;
+opt_type(queue_dir) ->
+    fun iolist_to_binary/1;
+opt_type(queue_type) ->
+    fun(ram) -> ram; (file) -> file end;
 opt_type(_) ->
-    [hide_sensitive_log_data, hosts, language,
-     default_db, default_ram_db, loglevel].
+    [hide_sensitive_log_data, hosts, language, max_fsm_queue,
+     default_db, default_ram_db, queue_type, queue_dir, loglevel].
 
 -spec may_hide_data(any()) -> any().
 may_hide_data(Data) ->
@@ -1486,3 +1491,11 @@ fsm_limit_opts(Opts) ->
 		N -> [{max_queue, N}]
 	    end
     end.
+
+-spec queue_dir() -> binary() | undefined.
+queue_dir() ->
+    get_option(queue_dir, opt_type(queue_dir)).
+
+-spec default_queue_type(binary()) -> ram | file.
+default_queue_type(Host) ->
+    get_option({queue_type, Host}, opt_type(queue_type), ram).

src/ejabberd_hooks.erl

@@ -189,7 +189,7 @@ run_fold(Hook, Host, Val, Args) ->
 %%          {stop, Reason}
 %%----------------------------------------------------------------------
 init([]) ->
-    ets:new(hooks, [named_table]),
+    ets:new(hooks, [named_table, {read_concurrency, true}]),
     {ok, #state{}}.
 
 %%----------------------------------------------------------------------

src/ejabberd_http.erl

@@ -68,6 +68,7 @@
 		end_of_request = false,
 		options = [],
 		default_host,
+		custom_headers,
 		trail = <<>>,
 		addr_re
 	       }).
@@ -167,10 +168,15 @@ init({SockMod, Socket}, Opts) ->
     DefaultHost = gen_mod:get_opt(default_host, Opts, fun(A) -> A end, undefined),
     {ok, RE} = re:compile(<<"^(?:\\[(.*?)\\]|(.*?))(?::(\\d+))?$">>),
 
+    CustomHeaders = gen_mod:get_opt(custom_headers, Opts,
+				    fun expand_custom_headers/1,
+				    []),
+
     ?INFO_MSG("started: ~p", [{SockMod1, Socket1}]),
     State = #state{sockmod = SockMod1,
                    socket = Socket1,
                    default_host = DefaultHost,
+		   custom_headers = CustomHeaders,
 		   options = Opts,
 		   request_handlers = RequestHandlers,
 		   addr_re = RE},
@@ -252,7 +258,7 @@ process_header(State, Data) ->
 		      request_version = Version, request_path = Path,
 		      request_keepalive = KeepAlive};
       {ok, {http_header, _, 'Connection' = Name, _, Conn}} ->
-	  KeepAlive1 = case jlib:tolower(Conn) of
+	  KeepAlive1 = case misc:tolower(Conn) of
 			 <<"keep-alive">> -> true;
 			 <<"close">> -> false;
 			 _ -> State#state.request_keepalive
@@ -309,6 +315,7 @@ process_header(State, Data) ->
 		       trail = State3#state.trail,
 		       options = State#state.options,
 		       default_host = State#state.default_host,
+		       custom_headers = State#state.custom_headers,
 		       request_handlers = State#state.request_handlers,
 		       addr_re = State#state.addr_re};
 	    _ ->
@@ -316,6 +323,7 @@ process_header(State, Data) ->
 		       trail = State3#state.trail,
 		       options = State#state.options,
 		       default_host = State#state.default_host,
+		       custom_headers = State#state.custom_headers,
 		       request_handlers = State#state.request_handlers,
 		       addr_re = State#state.addr_re}
 	  end;
@@ -323,6 +331,7 @@ process_header(State, Data) ->
 	  #state{end_of_request = true,
 		 options = State#state.options,
 		 default_host = State#state.default_host,
+		 custom_headers = State#state.custom_headers,
 		 request_handlers = State#state.request_handlers,
 		 addr_re = State#state.addr_re}
     end.
@@ -446,6 +455,7 @@ process_request(#state{request_method = Method,
 		       request_tp = TP,
 		       request_headers = RequestHeaders,
 		       request_handlers = RequestHandlers,
+		       custom_headers = CustomHeaders,
 		       trail = Trail} = State) ->
     case extract_path_query(State) of
 	{State2, false} ->
@@ -478,18 +488,21 @@ process_request(#state{request_method = Method,
                                ip = IP},
 	    Res = case process(RequestHandlers, Request, Socket, SockMod, Trail) of
 		      El when is_record(El, xmlel) ->
-			  make_xhtml_output(State, 200, [], El);
+			  make_xhtml_output(State, 200, CustomHeaders, El);
 		      {Status, Headers, El}
 			when is_record(El, xmlel) ->
-			  make_xhtml_output(State, Status, Headers, El);
+			  make_xhtml_output(State, Status,
+					    Headers ++ CustomHeaders, El);
 		      Output when is_binary(Output) or is_list(Output) ->
-			  make_text_output(State, 200, [], Output);
+			  make_text_output(State, 200, CustomHeaders, Output);
 		      {Status, Headers, Output}
 			when is_binary(Output) or is_list(Output) ->
-			  make_text_output(State, Status, Headers, Output);
+			  make_text_output(State, Status,
+					   Headers ++ CustomHeaders, Output);
 		      {Status, Reason, Headers, Output}
 			when is_binary(Output) or is_list(Output) ->
-			  make_text_output(State, Status, Reason, Headers, Output);
+			  make_text_output(State, Status, Reason,
+					   Headers ++ CustomHeaders, Output);
 		      _ ->
 			  none
 		  end,
@@ -497,7 +510,7 @@ process_request(#state{request_method = Method,
     end.
 
 make_bad_request(State) ->
-    make_xhtml_output(State, 400, [],
+    make_xhtml_output(State, 400, State#state.custom_headers,
 		      ejabberd_web:make_xhtml([#xmlel{name = <<"h1">>,
 						      attrs = [],
 						      children =
@@ -507,7 +520,7 @@ make_bad_request(State) ->
 analyze_ip_xff(IP, [], _Host) -> IP;
 analyze_ip_xff({IPLast, Port}, XFF, Host) ->
     [ClientIP | ProxiesIPs] = str:tokens(XFF, <<", ">>) ++
-				[jlib:ip_to_list(IPLast)],
+				[misc:ip_to_list(IPLast)],
     TrustedProxies = ejabberd_config:get_option(
                        {trusted_proxies, Host},
                        fun(all) -> all;
@@ -720,6 +733,11 @@ rest_dir(0, Path, <<H, T/binary>>) ->
     rest_dir(0, <<H, Path/binary>>, T);
 rest_dir(N, Path, <<_H, T/binary>>) -> rest_dir(N, Path, T).
 
+expand_custom_headers(Headers) ->
+    lists:map(fun({K, V}) ->
+		      {K, misc:expand_keyword(<<"@VERSION@">>, V, ?VERSION)}
+	      end, Headers).
+
 %% hex_to_integer
 
 hex_to_integer(Hex) ->
@@ -783,7 +801,7 @@ code_to_phrase(505) -> <<"HTTP Version Not Supported">>.
 
 -spec parse_auth(binary()) -> {binary(), binary()} | {oauth, binary(), []} | undefined.
 parse_auth(<<"Basic ", Auth64/binary>>) ->
-    Auth = jlib:decode_base64(Auth64),
+    Auth = misc:decode_base64(Auth64),
     %% Auth should be a string with the format: user@server:password
     %% Note that password can contain additional characters '@' and ':'
     case str:chr(Auth, $:) of

src/ejabberd_http_ws.erl

@@ -283,7 +283,7 @@ cancel_timer(Timer) ->
     receive {timeout, Timer, _} -> ok after 0 -> ok end.
 
 get_human_html_xmlel() ->
-    Heading = <<"ejabberd ", (jlib:atom_to_binary(?MODULE))/binary>>,
+    Heading = <<"ejabberd ", (misc:atom_to_binary(?MODULE))/binary>>,
     #xmlel{name = <<"html">>,
            attrs =
                [{<<"xmlns">>, <<"http://www.w3.org/1999/xhtml">>}],

src/ejabberd_listener.erl

@@ -206,10 +206,10 @@ parse_listener_portip(PortIP, Opts) ->
 	case add_proto(PortIP, Opts) of
 	    {P, Prot} ->
 		T = get_ip_tuple(IPOpt, IPVOpt),
-		S = jlib:ip_to_list(T),
+		S = misc:ip_to_list(T),
 		{P, T, S, Prot};
 	    {P, T, Prot} when is_integer(P) and is_tuple(T) ->
-		S = jlib:ip_to_list(T),
+		S = misc:ip_to_list(T),
 		{P, T, S, Prot};
 	    {P, S, Prot} when is_integer(P) and is_binary(S) ->
 		[S | _] = str:tokens(S, <<"/">>),

src/ejabberd_local.erl

@@ -197,10 +197,12 @@ get_features(_, _, XMLNSs) ->
 %%====================================================================
 
 init([]) ->
+    process_flag(trap_exit, true),
     lists:foreach(fun host_up/1, ?MYHOSTS),
     ejabberd_hooks:add(host_up, ?MODULE, host_up, 10),
     ejabberd_hooks:add(host_down, ?MODULE, host_down, 100),
-    catch ets:new(?IQTABLE, [named_table, public, ordered_set]),
+    catch ets:new(?IQTABLE, [named_table, public, ordered_set,
+			     {read_concurrency, true}]),
     update_table(),
     ejabberd_mnesia:create(?MODULE, iq_response,
 			[{ram_copies, [node()]},
@@ -288,7 +290,11 @@ host_up(Host) ->
 		       ?MODULE, bounce_resource_packet, 100).
 
 host_down(Host) ->
-    ejabberd_router:unregister_route(Host),
+    Owner = case whereis(?MODULE) of
+		undefined -> self();
+		Pid -> Pid
+	    end,
+    ejabberd_router:unregister_route(Host, Owner),
     ejabberd_hooks:delete(local_send_to_resource_hook, Host,
 			  ?MODULE, bounce_resource_packet, 100).
 

src/ejabberd_oauth.erl

@@ -83,21 +83,23 @@ get_commands_spec() ->
                         result = {result, {tuple, [{token, string}, {scopes, string}, {expires_in, string}]}}
                        },
      #ejabberd_commands{name = oauth_list_tokens, tags = [oauth],
-                        desc = "List oauth tokens, their user and scope, and how many seconds remain until expirity",
+                        desc = "List oauth tokens, user, scope, and seconds to expire (only Mnesia)",
+                        longdesc = "List oauth tokens, their user and scope, and how many seconds remain until expirity",
                         module = ?MODULE, function = oauth_list_tokens,
                         args = [],
                         policy = restricted,
                         result = {tokens, {list, {token, {tuple, [{token, string}, {user, string}, {scope, string}, {expires_in, string}]}}}}
                        },
      #ejabberd_commands{name = oauth_list_scopes, tags = [oauth],
-                        desc = "List scopes that can be granted to tokens generated through the command line, together with the commands they allow",
+                        desc = "List scopes that can be granted, and commands",
+                        longdesc = "List scopes that can be granted to tokens generated through the command line, together with the commands they allow",
                         module = ?MODULE, function = oauth_list_scopes,
                         args = [],
                         policy = restricted,
                         result = {scopes, {list, {scope, {tuple, [{scope, string}, {commands, string}]}}}}
                        },
      #ejabberd_commands{name = oauth_revoke_token, tags = [oauth],
-                        desc = "Revoke authorization for a token",
+                        desc = "Revoke authorization for a token (only Mnesia)",
                         module = ?MODULE, function = oauth_revoke_token,
                         args = [{token, string}],
                         policy = restricted,

src/ejabberd_redis.erl

@@ -23,12 +23,19 @@
 %%%----------------------------------------------------------------------
 
 -module(ejabberd_redis).
+-ifndef(GEN_SERVER).
+-define(GEN_SERVER, gen_server).
+-endif.
+-behaviour(?GEN_SERVER).
 
--behaviour(gen_server).
--behaviour(ejabberd_config).
+-compile({no_auto_import, [get/1, put/2]}).
 
 %% API
--export([start_link/0, q/1, qp/1, config_reloaded/0, opt_type/1]).
+-export([start_link/1, get_proc/1, q/1, qp/1, format_error/1]).
+%% Commands
+-export([multi/1, get/1, set/2, del/1,
+	 sadd/2, srem/2, smembers/1, sismember/2, scard/1,
+	 hget/2, hset/3, hdel/2, hlen/1, hgetall/1, hkeys/1]).
 
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
@@ -36,89 +43,311 @@
 
 -define(SERVER, ?MODULE).
 -define(PROCNAME, 'ejabberd_redis_client').
+-define(TR_STACK, redis_transaction_stack).
+-define(DEFAULT_MAX_QUEUE, 5000).
+-define(MAX_RETRIES, 1).
+-define(CALL_TIMEOUT, 60*1000). %% 60 seconds
 
 -include("logger.hrl").
 -include("ejabberd.hrl").
 
--record(state, {connection :: {pid(), reference()} | undefined}).
+-record(state, {connection :: pid() | undefined,
+		num :: pos_integer(),
+		pending_q :: p1_queue:queue()}).
+
+-type redis_error() :: {error, binary() | timeout | disconnected | overloaded}.
+-type redis_reply() :: binary() | [binary()].
+-type redis_command() :: [binary()].
+-type redis_pipeline() :: [redis_command()].
+-type state() :: #state{}.
 
 %%%===================================================================
 %%% API
 %%%===================================================================
-start_link() ->
-    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+start_link(I) ->
+    ?GEN_SERVER:start_link({local, get_proc(I)}, ?MODULE, [I], []).
 
+get_proc(I) ->
+    misc:binary_to_atom(
+      iolist_to_binary(
+	[atom_to_list(?MODULE), $_, integer_to_list(I)])).
+
+get_connection(I) ->
+    misc:binary_to_atom(
+      iolist_to_binary(
+	[atom_to_list(?MODULE), "_connection_", integer_to_list(I)])).
+
+-spec q(redis_command()) -> {ok, redis_reply()} | redis_error().
 q(Command) ->
-    try eredis:q(?PROCNAME, Command)
-    catch _:Reason -> {error, Reason}
-    end.
+    call(get_worker(), {q, Command}, ?MAX_RETRIES).
 
+-spec qp(redis_pipeline()) -> {ok, [redis_reply()]} | redis_error().
 qp(Pipeline) ->
-    try eredis:qp(?PROCNAME, Pipeline)
-    catch _:Reason -> {error, Reason}
+    call(get_worker(), {qp, Pipeline}, ?MAX_RETRIES).
+
+-spec multi(fun(() -> any())) -> {ok, [redis_reply()]} | redis_error().
+multi(F) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    erlang:put(?TR_STACK, []),
+	    try F() of
+		_ ->
+		    Stack = erlang:get(?TR_STACK),
+		    erlang:erase(?TR_STACK),
+		    Command = [["MULTI"]|lists:reverse([["EXEC"]|Stack])],
+		    case qp(Command) of
+			{error, _} = Err -> Err;
+			Result -> get_result(Result)
+		    end
+	    catch E:R ->
+		    erlang:erase(?TR_STACK),
+		    erlang:raise(E, R, erlang:get_stacktrace())
+	    end;
+	_ ->
+	    erlang:error(nested_transaction)
     end.
 
-config_reloaded() ->
-    case is_redis_configured() of
-	true ->
-	    ?MODULE ! connect;
-	false ->
-	    ?MODULE ! disconnect
+-spec format_error(atom() | binary()) -> binary().
+format_error(Reason) when is_atom(Reason) ->
+    format_error(misc:atom_to_binary(Reason));
+format_error(Reason) ->
+    Reason.
+
+%%%===================================================================
+%%% Redis commands API
+%%%===================================================================
+-spec get(iodata()) -> {ok, undefined | binary()} | redis_error().
+get(Key) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    q([<<"GET">>, Key]);
+	_ ->
+	    erlang:error(transaction_unsupported)
+    end.
+
+-spec set(iodata(), iodata()) -> ok | redis_error() | queued.
+set(Key, Val) ->
+    Cmd = [<<"SET">>, Key, Val],
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q(Cmd) of
+		{ok, <<"OK">>} -> ok;
+		{error, _} = Err -> Err
+	    end;
+	Stack ->
+	    tr_enq(Cmd, Stack)
+    end.
+
+-spec del(list()) -> {ok, non_neg_integer()} | redis_error() | queued.
+del([]) ->
+    reply(0);
+del(Keys) ->
+    Cmd = [<<"DEL">>|Keys],
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q(Cmd) of
+		{ok, N} -> {ok, binary_to_integer(N)};
+		{error, _} = Err -> Err
+	    end;
+	Stack ->
+	    tr_enq(Cmd, Stack)
+    end.
+
+-spec sadd(iodata(), list()) -> {ok, non_neg_integer()} | redis_error() | queued.
+sadd(_Set, []) ->
+    reply(0);
+sadd(Set, Members) ->
+    Cmd = [<<"SADD">>, Set|Members],
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q(Cmd) of
+		{ok, N} -> {ok, binary_to_integer(N)};
+		{error, _} = Err -> Err
+	    end;
+	Stack ->
+	    tr_enq(Cmd, Stack)
+    end.
+
+-spec srem(iodata(), list()) -> {ok, non_neg_integer()} | redis_error() | queued.
+srem(_Set, []) ->
+    reply(0);
+srem(Set, Members) ->
+    Cmd = [<<"SREM">>, Set|Members],
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q(Cmd) of
+		{ok, N} -> {ok, binary_to_integer(N)};
+		{error, _} = Err -> Err
+	    end;
+	Stack ->
+	    tr_enq(Cmd, Stack)
+    end.
+
+-spec smembers(iodata()) -> {ok, [binary()]} | redis_error().
+smembers(Set) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    q([<<"SMEMBERS">>, Set]);
+	_ ->
+	    erlang:error(transaction_unsupported)
+    end.
+
+-spec sismember(iodata(), iodata()) -> boolean() | redis_error().
+sismember(Set, Member) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q([<<"SISMEMBER">>, Set, Member]) of
+		{ok, Flag} -> {ok, dec_bool(Flag)};
+		{error, _} = Err -> Err
+	    end;
+	_ ->
+	    erlang:error(transaction_unsupported)
+    end.
+
+-spec scard(iodata()) -> {ok, non_neg_integer()} | redis_error().
+scard(Set) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q([<<"SCARD">>, Set]) of
+		{ok, N} ->
+		    {ok, binary_to_integer(N)};
+		{error, _} = Err ->
+		    Err
+	    end;
+	_ ->
+	    erlang:error(transaction_unsupported)
+    end.
+
+-spec hget(iodata(), iodata()) -> {ok, undefined | binary()} | redis_error().
+hget(Key, Field) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    q([<<"HGET">>, Key, Field]);
+	_ ->
+	    erlang:error(transaction_unsupported)
+    end.
+
+-spec hset(iodata(), iodata(), iodata()) -> {ok, boolean()} | redis_error() | queued.
+hset(Key, Field, Val) ->
+    Cmd = [<<"HSET">>, Key, Field, Val],
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q(Cmd) of
+		{ok, Flag} -> {ok, dec_bool(Flag)};
+		{error, _} = Err -> Err
+	    end;
+	Stack ->
+	    tr_enq(Cmd, Stack)
+    end.
+
+-spec hdel(iodata(), list()) -> {ok, non_neg_integer()} | redis_error() | queued.
+hdel(_Key, []) ->
+    reply(0);
+hdel(Key, Fields) ->
+    Cmd = [<<"HDEL">>, Key|Fields],
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q(Cmd) of
+		{ok, N} -> {ok, binary_to_integer(N)};
+		{error, _} = Err -> Err
+	    end;
+	Stack ->
+	    tr_enq(Cmd, Stack)
+    end.
+
+-spec hgetall(iodata()) -> {ok, [{binary(), binary()}]} | redis_error().
+hgetall(Key) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q([<<"HGETALL">>, Key]) of
+		{ok, Pairs} -> {ok, decode_pairs(Pairs)};
+		{error, _} = Err -> Err
+	    end;
+	_ ->
+	    erlang:error(transaction_unsupported)
+    end.
+
+-spec hlen(iodata()) -> {ok, non_neg_integer()} | redis_error().
+hlen(Key) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    case q([<<"HLEN">>, Key]) of
+		{ok, N} -> {ok, binary_to_integer(N)};
+		{error, _} = Err -> Err
+	    end;
+	_ ->
+	    erlang:error(transaction_unsupported)
+    end.
+
+-spec hkeys(iodata()) -> {ok, [binary()]} | redis_error().
+hkeys(Key) ->
+    case erlang:get(?TR_STACK) of
+	undefined ->
+	    q([<<"HKEYS">>, Key]);
+	_ ->
+	    erlang:error(transaction_unsupported)
     end.
 
 %%%===================================================================
 %%% gen_server callbacks
 %%%===================================================================
-init([]) ->
-    ejabberd_hooks:add(config_reloaded, ?MODULE, config_reloaded, 20),
+init([I]) ->
     process_flag(trap_exit, true),
+    QueueType = get_queue_type(),
+    Limit = max_fsm_queue(),
     self() ! connect,
-    {ok, #state{}}.
+    {ok, #state{num = I, pending_q = p1_queue:new(QueueType, Limit)}}.
 
-handle_call(_Request, _From, State) ->
-    Reply = ok,
-    {reply, Reply, State}.
+handle_call(connect, From, #state{connection = undefined,
+				  pending_q = Q} = State) ->
+    CurrTime = p1_time_compat:monotonic_time(milli_seconds),
+    Q2 = try p1_queue:in({From, CurrTime}, Q)
+	 catch error:full ->
+		 Q1 = clean_queue(Q, CurrTime),
+		 p1_queue:in({From, CurrTime}, Q1)
+	 end,
+    {noreply, State#state{pending_q = Q2}};
+handle_call(connect, From, #state{connection = Pid} = State) ->
+    case is_process_alive(Pid) of
+	true ->
+	    {reply, ok, State};
+	false ->
+	    self() ! connect,
+	    handle_call(connect, From, State#state{connection = undefined})
+    end;
+handle_call(Request, _From, State) ->
+    ?WARNING_MSG("unexepected call: ~p", [Request]),
+    {noreply, State}.
 
 handle_cast(_Msg, State) ->
     {noreply, State}.
 
 handle_info(connect, #state{connection = undefined} = State) ->
-    NewState = case is_redis_configured() of
-		   true ->
-		       case connect() of
-			   {ok, Connection} ->
-			       State#state{connection = Connection};
-			   {error, _} ->
-			       State
-		       end;
-		   false ->
+    NewState = case connect(State) of
+		   {ok, Connection} ->
+		       Q1 = flush_queue(State#state.pending_q),
+		       State#state{connection = Connection, pending_q = Q1};
+		   {error, _} ->
 		       State
 	       end,
     {noreply, NewState};
 handle_info(connect, State) ->
     %% Already connected
     {noreply, State};
-handle_info(disconnect, #state{connection = {Pid, MRef}} = State) ->
-    ?INFO_MSG("Disconnecting from Redis server", []),
-    erlang:demonitor(MRef, [flush]),
-    eredis:stop(Pid),
-    {noreply, State#state{connection = undefined}};
-handle_info(disconnect, State) ->
-    %% Not connected
-    {noreply, State};
-handle_info({'DOWN', MRef, _Type, Pid, Reason},
-	    #state{connection = {Pid, MRef}} = State) ->
-    ?INFO_MSG("Redis connection has failed: ~p", [Reason]),
-    connect(),
-    {noreply, State#state{connection = undefined}};
-handle_info({'EXIT', _, _}, State) ->
-    {noreply, State};
+handle_info({'EXIT', Pid, _}, State) ->
+    case State#state.connection of
+	Pid ->
+	    self() ! connect,
+	    {noreply, State#state{connection = undefined}};
+	_ ->
+	    {noreply, State}
+    end;
 handle_info(Info, State) ->
-    ?INFO_MSG("unexpected info = ~p", [Info]),
+    ?WARNING_MSG("unexpected info = ~p", [Info]),
     {noreply, State}.
 
 terminate(_Reason, _State) ->
-    ejabberd_hooks:delete(config_reloaded, ?MODULE, config_reloaded, 20).
+    ok.
 
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
@@ -126,37 +355,8 @@ code_change(_OldVsn, State, _Extra) ->
 %%%===================================================================
 %%% Internal functions
 %%%===================================================================
-is_redis_configured() ->
-    lists:any(fun is_redis_configured/1, ?MYHOSTS).
-
-is_redis_configured(Host) ->
-    ServerConfigured = ejabberd_config:has_option({redis_server, Host}),
-    PortConfigured = ejabberd_config:has_option({redis_port, Host}),
-    DBConfigured = ejabberd_config:has_option({redis_db, Host}),
-    PassConfigured = ejabberd_config:has_option({redis_password, Host}),
-    ReconnTimeoutConfigured = ejabberd_config:has_option(
-				{redis_reconnect_timeout, Host}),
-    ConnTimeoutConfigured = ejabberd_config:has_option(
-			      {redis_connect_timeout, Host}),
-    Modules = ejabberd_config:get_option(
-		{modules, Host},
-		fun(L) when is_list(L) -> L end, []),
-    SMConfigured = ejabberd_config:get_option(
-		     {sm_db_type, Host},
-		     fun(V) -> V end) == redis,
-    ModuleWithRedisDBConfigured =
-	lists:any(
-	  fun({Module, Opts}) ->
-		  gen_mod:db_type(Host, Opts, Module) == redis
-	  end, Modules),
-    ServerConfigured or PortConfigured or DBConfigured or PassConfigured or
-	ReconnTimeoutConfigured or ConnTimeoutConfigured or
-	SMConfigured or ModuleWithRedisDBConfigured.
-
-iolist_to_list(IOList) ->
-    binary_to_list(iolist_to_binary(IOList)).
-
-connect() ->
+-spec connect(state()) -> {ok, pid()} | {error, any()}.
+connect(#state{num = Num}) ->
     Server = ejabberd_config:get_option(redis_server,
 					fun iolist_to_list/1,
 					"localhost"),
@@ -172,46 +372,162 @@ connect() ->
     Pass = ejabberd_config:get_option(redis_password,
 				      fun iolist_to_list/1,
 				      ""),
-    ReconnTimeout = timer:seconds(
-		      ejabberd_config:get_option(
-			redis_reconnect_timeout,
-			fun(I) when is_integer(I), I>0 -> I end,
-			1)),
     ConnTimeout = timer:seconds(
 		    ejabberd_config:get_option(
 		      redis_connect_timeout,
 		      fun(I) when is_integer(I), I>0 -> I end,
 		      1)),
     try case eredis:start_link(Server, Port, DB, Pass,
-			       ReconnTimeout, ConnTimeout) of
+			       no_reconnect, ConnTimeout) of
 	    {ok, Client} ->
-		?INFO_MSG("Connected to Redis at ~s:~p", [Server, Port]),
-		unlink(Client),
-		MRef = erlang:monitor(process, Client),
-		register(?PROCNAME, Client),
-		{ok, {Client, MRef}};
+		?DEBUG("Connection #~p established to Redis at ~s:~p",
+		       [Num, Server, Port]),
+		register(get_connection(Num), Client),
+		{ok, Client};
 	    {error, Why} ->
 		erlang:error(Why)
 	end
     catch _:Reason ->
-	    Timeout = 10,
-	    ?ERROR_MSG("Redis connection at ~s:~p has failed: ~p; "
+	    Timeout = randoms:uniform(
+			min(10, ejabberd_redis_sup:get_pool_size())),
+	    ?ERROR_MSG("Redis connection #~p at ~s:~p has failed: ~p; "
 		       "reconnecting in ~p seconds",
-		       [Server, Port, Reason, Timeout]),
+		       [Num, Server, Port, Reason, Timeout]),
 	    erlang:send_after(timer:seconds(Timeout), self(), connect),
 	    {error, Reason}
     end.
 
-opt_type(redis_connect_timeout) ->
-    fun (I) when is_integer(I), I > 0 -> I end;
-opt_type(redis_db) ->
-    fun (I) when is_integer(I), I >= 0 -> I end;
-opt_type(redis_password) -> fun iolist_to_list/1;
-opt_type(redis_port) ->
-    fun (P) when is_integer(P), P > 0, P < 65536 -> P end;
-opt_type(redis_reconnect_timeout) ->
-    fun (I) when is_integer(I), I > 0 -> I end;
-opt_type(redis_server) -> fun iolist_to_list/1;
-opt_type(_) ->
-    [redis_connect_timeout, redis_db, redis_password,
-     redis_port, redis_reconnect_timeout, redis_server].
+-spec call({atom(), atom()}, {q, redis_command()}, integer()) ->
+		  {ok, redis_reply()} | redis_error();
+	  ({atom(), atom()}, {qp, redis_pipeline()}, integer()) ->
+		  {ok, [redis_reply()]} | redis_error().
+call({Conn, Parent}, {F, Cmd}, Retries) ->
+    ?DEBUG("redis query: ~p", [Cmd]),
+    Res = try eredis:F(Conn, Cmd, ?CALL_TIMEOUT) of
+	      {error, Reason} when is_atom(Reason) ->
+		  try exit(whereis(Conn), kill) catch _:_ -> ok end,
+		  {error, disconnected};
+	      Other ->
+		  Other
+	  catch exit:{timeout, _} -> {error, timeout};
+		exit:{_, {gen_server, call, _}} -> {error, disconnected}
+	  end,
+    case Res of
+	{error, disconnected} when Retries > 0 ->
+	    try ?GEN_SERVER:call(Parent, connect, ?CALL_TIMEOUT) of
+		ok -> call({Conn, Parent}, {F, Cmd}, Retries-1);
+		{error, _} = Err -> Err
+	    catch exit:{Why, {?GEN_SERVER, call, _}} ->
+		    Reason1 = case Why of
+				 timeout -> timeout;
+				 _ -> disconnected
+			     end,
+		    log_error(Cmd, Reason1),
+		    {error, Reason1}
+	    end;
+	{error, Reason1} ->
+	    log_error(Cmd, Reason1),
+	    Res;
+	_ ->
+	    Res
+    end.
+
+-spec log_error(redis_command() | redis_pipeline(), atom() | binary()) -> ok.
+log_error(Cmd, Reason) ->
+    ?ERROR_MSG("Redis request has failed:~n"
+	       "** request = ~p~n"
+	       "** response = ~s",
+	       [Cmd, format_error(Reason)]).
+
+-spec get_worker() -> {atom(), atom()}.
+get_worker() ->
+    Time = p1_time_compat:system_time(),
+    I = erlang:phash2(Time, ejabberd_redis_sup:get_pool_size()) + 1,
+    {get_connection(I), get_proc(I)}.
+
+-spec get_result([{error, atom() | binary()} | {ok, iodata()}]) ->
+			{ok, [redis_reply()]} | {error, binary()}.
+get_result([{error, _} = Err|_]) ->
+    Err;
+get_result([{ok, _} = OK]) ->
+    OK;
+get_result([_|T]) ->
+    get_result(T).
+
+-spec tr_enq([iodata()], list()) -> queued.
+tr_enq(Cmd, Stack) ->
+    erlang:put(?TR_STACK, [Cmd|Stack]),
+    queued.
+
+-spec decode_pairs([binary()]) -> [{binary(), binary()}].
+decode_pairs(Pairs) ->
+    decode_pairs(Pairs, []).
+
+-spec decode_pairs([binary()], [{binary(), binary()}]) -> [{binary(), binary()}].
+decode_pairs([Field, Val|Pairs], Acc) ->
+    decode_pairs(Pairs, [{Field, Val}|Acc]);
+decode_pairs([], Acc) ->
+    lists:reverse(Acc).
+
+dec_bool(<<$1>>) -> true;
+dec_bool(<<$0>>) -> false.
+
+-spec reply(T) -> {ok, T} | queued.
+reply(Val) ->
+    case erlang:get(?TR_STACK) of
+	undefined -> {ok, Val};
+	_ -> queued
+    end.
+
+-spec iolist_to_list(iodata()) -> string().
+iolist_to_list(IOList) ->
+    binary_to_list(iolist_to_binary(IOList)).
+
+-spec max_fsm_queue() -> pos_integer().
+max_fsm_queue() ->
+    proplists:get_value(max_queue, fsm_limit_opts(), ?DEFAULT_MAX_QUEUE).
+
+fsm_limit_opts() ->
+    ejabberd_config:fsm_limit_opts([]).
+
+get_queue_type() ->
+    case ejabberd_config:get_option(
+	   redis_queue_type,
+	   ejabberd_redis_sup:opt_type(redis_queue_type)) of
+	undefined ->
+	    ejabberd_config:default_queue_type(global);
+	Type ->
+	    Type
+    end.
+
+-spec flush_queue(p1_queue:queue()) -> p1_queue:queue().
+flush_queue(Q) ->
+    CurrTime = p1_time_compat:monotonic_time(milli_seconds),
+    p1_queue:dropwhile(
+      fun({From, Time}) ->
+	      if (CurrTime - Time) >= ?CALL_TIMEOUT ->
+		      ok;
+		 true ->
+		      ?GEN_SERVER:reply(From, ok)
+	      end,
+	      true
+      end, Q).
+
+-spec clean_queue(p1_queue:queue(), integer()) -> p1_queue:queue().
+clean_queue(Q, CurrTime) ->
+    Q1 = p1_queue:dropwhile(
+	   fun({_From, Time}) ->
+		   (CurrTime - Time) >= ?CALL_TIMEOUT
+	   end, Q),
+    Len = p1_queue:len(Q1),
+    Limit = p1_queue:get_limit(Q1),
+    if Len >= Limit ->
+	    ?ERROR_MSG("Redis request queue is overloaded", []),
+	    p1_queue:dropwhile(
+	      fun({From, _Time}) ->
+		      ?GEN_SERVER:reply(From, {error, overloaded}),
+		      true
+	      end, Q1);
+       true ->
+	    Q1
+    end.

src/ejabberd_redis_sup.erl

@@ -0,0 +1,159 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% Created :  6 Apr 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2017   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(ejabberd_redis_sup).
+
+-behaviour(supervisor).
+
+%% API
+-export([start_link/0, get_pool_size/0,
+	 host_up/1, config_reloaded/0, opt_type/1]).
+
+%% Supervisor callbacks
+-export([init/1]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+
+-define(DEFAULT_POOL_SIZE, 10).
+
+%%%===================================================================
+%%% API functions
+%%%===================================================================
+start_link() ->
+    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+host_up(Host) ->
+    case is_redis_configured(Host) of
+	true ->
+	    ejabberd:start_app(eredis),
+	    lists:foreach(
+	      fun(Spec) ->
+		      supervisor:start_child(?MODULE, Spec)
+	      end, get_specs());
+	false ->
+	    ok
+    end.
+
+config_reloaded() ->
+    case is_redis_configured() of
+	true ->
+	    ejabberd:start_app(eredis),
+	    lists:foreach(
+	      fun(Spec) ->
+		      supervisor:start_child(?MODULE, Spec)
+	      end, get_specs()),
+	    PoolSize = get_pool_size(),
+	    lists:foreach(
+	      fun({Id, _, _, _}) when Id > PoolSize ->
+		      supervisor:terminate_child(?MODULE, Id),
+		      supervisor:delete_child(?MODULE, Id);
+		 (_) ->
+		      ok
+	      end, supervisor:which_children(?MODULE));
+	false ->
+	    lists:foreach(
+	      fun({Id, _, _, _}) ->
+		      supervisor:terminate_child(?MODULE, Id),
+		      supervisor:delete_child(?MODULE, Id)
+	      end, supervisor:which_children(?MODULE))
+    end.
+
+%%%===================================================================
+%%% Supervisor callbacks
+%%%===================================================================
+init([]) ->
+    ejabberd_hooks:add(config_reloaded, ?MODULE, config_reloaded, 20),
+    ejabberd_hooks:add(host_up, ?MODULE, host_up, 20),
+    Specs = case is_redis_configured() of
+		true ->
+		    ejabberd:start_app(eredis),
+		    get_specs();
+		false ->
+		    []
+	    end,
+    {ok, {{one_for_one, 500, 1}, Specs}}.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+is_redis_configured() ->
+    lists:any(fun is_redis_configured/1, ?MYHOSTS).
+
+is_redis_configured(Host) ->
+    ServerConfigured = ejabberd_config:has_option({redis_server, Host}),
+    PortConfigured = ejabberd_config:has_option({redis_port, Host}),
+    DBConfigured = ejabberd_config:has_option({redis_db, Host}),
+    PassConfigured = ejabberd_config:has_option({redis_password, Host}),
+    PoolSize = ejabberd_config:has_option({redis_pool_size, Host}),
+    ConnTimeoutConfigured = ejabberd_config:has_option(
+			      {redis_connect_timeout, Host}),
+    Modules = ejabberd_config:get_option(
+		{modules, Host},
+		fun(L) when is_list(L) -> L end, []),
+    SMConfigured = ejabberd_config:get_option(
+		     {sm_db_type, Host},
+		     fun(V) -> V end) == redis,
+    RouterConfigured = ejabberd_config:get_option(
+			 {router_db_type, Host},
+			 fun(V) -> V end) == redis,
+    ModuleWithRedisDBConfigured =
+	lists:any(
+	  fun({Module, Opts}) ->
+		  gen_mod:db_type(Host, Opts, Module) == redis
+	  end, Modules),
+    ServerConfigured or PortConfigured or DBConfigured or PassConfigured or
+	PoolSize or ConnTimeoutConfigured or
+	SMConfigured or RouterConfigured or ModuleWithRedisDBConfigured.
+
+get_specs() ->
+    lists:map(
+      fun(I) ->
+	      {I, {ejabberd_redis, start_link, [I]},
+	       transient, 2000, worker, [?MODULE]}
+      end, lists:seq(1, get_pool_size())).
+
+get_pool_size() ->
+    ejabberd_config:get_option(
+      redis_pool_size,
+      fun(N) when is_integer(N), N >= 1 -> N end,
+      ?DEFAULT_POOL_SIZE).
+
+iolist_to_list(IOList) ->
+    binary_to_list(iolist_to_binary(IOList)).
+
+opt_type(redis_connect_timeout) ->
+    fun (I) when is_integer(I), I > 0 -> I end;
+opt_type(redis_db) ->
+    fun (I) when is_integer(I), I >= 0 -> I end;
+opt_type(redis_password) -> fun iolist_to_list/1;
+opt_type(redis_port) ->
+    fun (P) when is_integer(P), P > 0, P < 65536 -> P end;
+opt_type(redis_server) -> fun iolist_to_list/1;
+opt_type(redis_pool_size) ->
+    fun (I) when is_integer(I), I > 0 -> I end;
+opt_type(redis_queue_type) ->
+    fun(ram) -> ram; (file) -> file end;
+opt_type(_) ->
+    [redis_connect_timeout, redis_db, redis_password,
+     redis_port, redis_pool_size, redis_server,
+     redis_pool_size, redis_queue_type].

src/ejabberd_riak.erl

@@ -84,7 +84,7 @@ is_connected() ->
 
 %% @private
 get_proc(I) ->
-    jlib:binary_to_atom(
+    misc:binary_to_atom(
       iolist_to_binary(
 	[atom_to_list(?MODULE), $_, integer_to_list(I)])).
 

src/ejabberd_riak_sup.erl

@@ -189,7 +189,7 @@ get_pids() ->
     [ejabberd_riak:get_proc(I) || I <- lists:seq(1, get_pool_size())].
 
 get_random_pid() ->
-    get_random_pid(p1_time_compat:monotonic_time()).
+    get_random_pid(p1_time_compat:system_time()).
 
 get_random_pid(Term) ->
     I = erlang:phash2(Term, get_pool_size()) + 1,

src/ejabberd_router.erl

@@ -44,10 +44,12 @@
 	 host_of_route/1,
 	 process_iq/1,
 	 unregister_route/1,
+	 unregister_route/2,
 	 unregister_routes/1,
 	 get_all_routes/0,
 	 is_my_route/1,
 	 is_my_host/1,
+	 find_routes/0,
 	 get_backend/0]).
 
 -export([start_link/0]).
@@ -67,8 +69,9 @@
 -callback init() -> any().
 -callback register_route(binary(), binary(), local_hint(),
 			 undefined | pos_integer(), pid()) -> ok | {error, term()}.
--callback unregister_route(binary(), undefined | pos_integer()) -> ok | {error, term()}.
+-callback unregister_route(binary(), undefined | pos_integer(), pid()) -> ok | {error, term()}.
 -callback find_routes(binary()) -> [#route{}].
+-callback find_routes() -> [#route{}].
 -callback host_of_route(binary()) -> {ok, binary()} | error.
 -callback is_my_route(binary()) -> boolean().
 -callback is_my_host(binary()) -> boolean().
@@ -171,12 +174,17 @@ register_routes(Domains) ->
 
 -spec unregister_route(binary()) -> ok.
 unregister_route(Domain) ->
+    unregister_route(Domain, self()).
+
+-spec unregister_route(binary(), pid()) -> ok.
+unregister_route(Domain, Pid) ->
     case jid:nameprep(Domain) of
 	error ->
 	    erlang:error({invalid_domain, Domain});
 	LDomain ->
 	    Mod = get_backend(),
-	    case Mod:unregister_route(LDomain, get_component_number(LDomain)) of
+	    case Mod:unregister_route(
+		   LDomain, get_component_number(LDomain), Pid) of
 		ok ->
 		    ?DEBUG("Route unregistered: ~s", [LDomain]);
 		{error, Err} ->
@@ -196,6 +204,11 @@ get_all_routes() ->
     Mod = get_backend(),
     Mod:get_all_routes().
 
+-spec find_routes() -> [#route{}].
+find_routes() ->
+    Mod = get_backend(),
+    Mod:find_routes().
+
 -spec host_of_route(binary()) -> binary().
 host_of_route(Domain) ->
     case jid:nameprep(Domain) of
@@ -333,8 +346,8 @@ get_component_number(LDomain) ->
 get_domain_balancing(From, To, LDomain) ->
     case ejabberd_config:get_option(
 	   {domain_balancing, LDomain}, fun(D) when is_atom(D) -> D end) of
-	undefined -> p1_time_compat:monotonic_time();
-	random -> p1_time_compat:monotonic_time();
+	undefined -> p1_time_compat:system_time();
+	random -> p1_time_compat:system_time();
 	source -> jid:tolower(From);
 	destination -> jid:tolower(To);
 	bare_source -> jid:remove_resource(jid:tolower(From));

src/ejabberd_router_mnesia.erl

@@ -24,8 +24,9 @@
 -behaviour(gen_server).
 
 %% API
--export([init/0, register_route/5, unregister_route/2, find_routes/1,
-	 host_of_route/1, is_my_route/1, is_my_host/1, get_all_routes/0]).
+-export([init/0, register_route/5, unregister_route/3, find_routes/1,
+	 host_of_route/1, is_my_route/1, is_my_host/1, get_all_routes/0,
+	 find_routes/0]).
 %% gen_server callbacks
 -export([init/1, handle_cast/2, handle_call/3, handle_info/2,
 	 terminate/2, code_change/3, start_link/0]).
@@ -96,19 +97,19 @@ register_route(Domain, ServerHost, _LocalHint, N, Pid) ->
 	end,
     transaction(F).
 
-unregister_route(Domain, undefined) ->
+unregister_route(Domain, undefined, Pid) ->
     F = fun () ->
 		case mnesia:match_object(
-		       #route{domain = Domain, pid = self(), _ = '_'}) of
+		       #route{domain = Domain, pid = Pid, _ = '_'}) of
 		    [R] -> mnesia:delete_object(R);
 		    _ -> ok
 		end
 	end,
     transaction(F);
-unregister_route(Domain, _) ->
+unregister_route(Domain, _, Pid) ->
     F = fun () ->
 		case mnesia:match_object(
-		       #route{domain = Domain, pid = self(), _ = '_'}) of
+		       #route{domain = Domain, pid = Pid, _ = '_'}) of
 		    [R] ->
 			I = R#route.local_hint,
 			ServerHost = R#route.server_host,
@@ -152,6 +153,9 @@ get_all_routes() ->
 	      when Domain /= ServerHost -> Domain
 	end)).
 
+find_routes() ->
+    ets:tab2list(route).
+
 %%%===================================================================
 %%% gen_server callbacks
 %%%===================================================================

src/ejabberd_router_redis.erl

@@ -0,0 +1,145 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% Created : 28 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2017   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(ejabberd_router_redis).
+-behaviour(ejabberd_router).
+
+%% API
+-export([init/0, register_route/5, unregister_route/3, find_routes/1,
+	 host_of_route/1, is_my_route/1, is_my_host/1, get_all_routes/0,
+	 find_routes/0]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+-include("ejabberd_router.hrl").
+
+-define(ROUTES_KEY, "ejabberd:routes").
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+init() ->
+    clean_table().
+
+register_route(Domain, ServerHost, LocalHint, _, Pid) ->
+    DomKey = domain_key(Domain),
+    PidKey = term_to_binary(Pid),
+    T = term_to_binary({ServerHost, LocalHint}),
+    case ejabberd_redis:multi(
+	   fun() ->
+		   ejabberd_redis:hset(DomKey, PidKey, T),
+		   ejabberd_redis:sadd(?ROUTES_KEY, [Domain])
+	   end) of
+	{ok, _} ->
+	    ok;
+	{error, _} ->
+	    {error, db_failure}
+    end.
+
+unregister_route(Domain, _, Pid) ->
+    DomKey = domain_key(Domain),
+    PidKey = term_to_binary(Pid),
+    try
+	{ok, Num} = ejabberd_redis:hdel(DomKey, [PidKey]),
+	if Num > 0 ->
+		{ok, Len} = ejabberd_redis:hlen(DomKey),
+		if Len == 0 ->
+			{ok, _} = ejabberd_redis:multi(
+				    fun() ->
+					    ejabberd_redis:del([DomKey]),
+					    ejabberd_redis:srem(?ROUTES_KEY, [Domain])
+				    end),
+			ok;
+		   true ->
+			ok
+		end;
+	   true ->
+		ok
+	end
+    catch _:{badmatch, {error, _}} ->
+	    {error, db_failure}
+    end.
+
+find_routes(Domain) ->
+    DomKey = domain_key(Domain),
+    case ejabberd_redis:hgetall(DomKey) of
+	{ok, Vals} ->
+	    decode_routes(Domain, Vals);
+	{error, _} ->
+	    []
+    end.
+
+host_of_route(Domain) ->
+    DomKey = domain_key(Domain),
+    case ejabberd_redis:hgetall(DomKey) of
+	{ok, [{_Pid, Data}|_]} ->
+	    {ServerHost, _} = binary_to_term(Data),
+	    {ok, ServerHost};
+	_ ->
+	    error
+    end.
+
+is_my_route(Domain) ->
+    case ejabberd_redis:sismember(?ROUTES_KEY, Domain) of
+	{ok, Bool} ->
+	    Bool;
+	{error, _} ->
+	    false
+    end.
+
+is_my_host(Domain) ->
+    {ok, Domain} == host_of_route(Domain).
+
+get_all_routes() ->
+    case ejabberd_redis:smembers(?ROUTES_KEY) of
+	{ok, Routes} ->
+	    Routes;
+	{error, _} ->
+	    []
+    end.
+
+find_routes() ->
+    lists:flatmap(fun find_routes/1, get_all_routes()).
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+clean_table() ->
+    lists:foreach(
+      fun(#route{domain = Domain, pid = Pid}) when node(Pid) == node() ->
+	      unregister_route(Domain, undefined, Pid);
+	 (_) ->
+	      ok
+      end, find_routes()).
+
+domain_key(Domain) ->
+    <<"ejabberd:route:", Domain/binary>>.
+
+decode_routes(Domain, Vals) ->
+    lists:map(
+      fun({Pid, Data}) ->
+	      {ServerHost, LocalHint} = binary_to_term(Data),
+	      #route{domain = Domain,
+		     pid = binary_to_term(Pid),
+		     server_host = ServerHost,
+		     local_hint = LocalHint}
+      end, Vals).

src/ejabberd_router_sql.erl

@@ -0,0 +1,170 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% Created : 28 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2017   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(ejabberd_router_sql).
+-behaviour(ejabberd_router).
+
+-compile([{parse_transform, ejabberd_sql_pt}]).
+
+%% API
+-export([init/0, register_route/5, unregister_route/3, find_routes/1,
+	 host_of_route/1, is_my_route/1, is_my_host/1, get_all_routes/0,
+	 find_routes/0]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+-include("ejabberd_sql_pt.hrl").
+-include("ejabberd_router.hrl").
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+init() ->
+    Node = erlang:atom_to_binary(node(), latin1),
+    ?INFO_MSG("Cleaning SQL 'route' table...", []),
+    case ejabberd_sql:sql_query(
+	   ?MYNAME, ?SQL("delete from route where node=%(Node)s")) of
+	{updated, _} ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to clean 'route' table: ~p", [Err]),
+	    Err
+    end.
+
+register_route(Domain, ServerHost, LocalHint, _, Pid) ->
+    PidS = misc:encode_pid(Pid),
+    LocalHintS = enc_local_hint(LocalHint),
+    Node = erlang:atom_to_binary(node(Pid), latin1),
+    case ?SQL_UPSERT(?MYNAME, "route",
+		     ["!domain=%(Domain)s",
+		      "!server_host=%(ServerHost)s",
+		      "!node=%(Node)s",
+		      "!pid=%(PidS)s",
+		      "local_hint=%(LocalHintS)s"]) of
+	ok ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to update 'route' table: ~p", [Err]),
+	    {error, Err}
+    end.
+
+unregister_route(Domain, _, Pid) ->
+    PidS = misc:encode_pid(Pid),
+    Node = erlang:atom_to_binary(node(Pid), latin1),
+    ejabberd_sql:sql_query(
+      ?MYNAME,
+      ?SQL("delete from route where domain=%(Domain)s "
+	   "and pid=%(PidS)s and node=%(Node)s")),
+    %% TODO: return meaningful error
+    ok.
+
+find_routes(Domain) ->
+    case ejabberd_sql:sql_query(
+	   ?MYNAME,
+	   ?SQL("select @(server_host)s, @(node)s, @(pid)s, @(local_hint)s "
+		"from route where domain=%(Domain)s")) of
+	{selected, Rows} ->
+	    lists:flatmap(
+	      fun(Row) ->
+		      row_to_route(Domain, Row)
+	      end, Rows);
+	Err ->
+	    ?ERROR_MSG("failed to select from 'route' table: ~p", [Err]),
+	    {error, Err}
+    end.
+
+host_of_route(Domain) ->
+    case ejabberd_sql:sql_query(
+	   ?MYNAME,
+	   ?SQL("select @(server_host)s from route where domain=%(Domain)s")) of
+	{selected, [{ServerHost}|_]} ->
+	    {ok, ServerHost};
+	{selected, []} ->
+	    error;
+	Err ->
+	    ?ERROR_MSG("failed to select from 'route' table: ~p", [Err]),
+	    error
+    end.
+
+is_my_route(Domain) ->
+    case host_of_route(Domain) of
+	{ok, _} -> true;
+	_ -> false
+    end.
+
+is_my_host(Domain) ->
+    {ok, Domain} == host_of_route(Domain).
+
+get_all_routes() ->
+    case ejabberd_sql:sql_query(
+	   ?MYNAME,
+	   ?SQL("select @(domain)s from route where domain <> server_host")) of
+	{selected, Domains} ->
+	    [Domain || {Domain} <- Domains];
+	Err ->
+	    ?ERROR_MSG("failed to select from 'route' table: ~p", [Err]),
+	    []
+    end.
+
+find_routes() ->
+    case ejabberd_sql:sql_query(
+	   ?MYNAME,
+	   ?SQL("select @(domain)s, @(server_host)s, @(node)s, @(pid)s, "
+		"@(local_hint)s from route")) of
+	{selected, Rows} ->
+	    lists:flatmap(
+	      fun({Domain, ServerHost, Node, Pid, LocalHint}) ->
+		      row_to_route(Domain, {ServerHost, Node, Pid, LocalHint})
+	      end, Rows);
+	Err ->
+	    ?ERROR_MSG("failed to select from 'route' table: ~p", [Err]),
+	    []
+    end.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+enc_local_hint(undefined) ->
+    <<"">>;
+enc_local_hint(LocalHint) ->
+    misc:term_to_expr(LocalHint).
+
+dec_local_hint(<<"">>) ->
+    undefined;
+dec_local_hint(S) ->
+    ejabberd_sql:decode_term(S).
+
+row_to_route(Domain, {ServerHost, NodeS, PidS, LocalHintS} = Row) ->
+    try	[#route{domain = Domain,
+		server_host = ServerHost,
+		pid = misc:decode_pid(PidS, NodeS),
+		local_hint = dec_local_hint(LocalHintS)}]
+    catch _:{bad_node, _} ->
+	    [];
+	  E:R ->
+	    ?ERROR_MSG("failed to decode row from 'route' table:~n"
+		       "Row = ~p~n"
+		       "Domain = ~s~n"
+		       "Reason = ~p",
+		       [Row, Domain, {E, {R, erlang:get_stacktrace()}}]),
+	    []
+    end.

src/ejabberd_s2s.erl

@@ -45,7 +45,7 @@
 	 external_host_overloaded/1, is_temporarly_blocked/1,
 	 get_commands_spec/0, zlib_enabled/1, get_idle_timeout/1,
 	 tls_required/1, tls_verify/1, tls_enabled/1, tls_options/2,
-	 host_up/1, host_down/1]).
+	 host_up/1, host_down/1, queue_type/1]).
 
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2,
@@ -285,6 +285,14 @@ get_idle_timeout(LServer) ->
 	 (infinity) -> infinity
       end, timer:minutes(10)).
 
+-spec queue_type(binary()) -> ram | file.
+queue_type(LServer) ->
+    case ejabberd_config:get_option(
+	   {s2s_queue_type, LServer}, opt_type(s2s_queue_type)) of
+	undefined -> ejabberd_config:default_queue_type(LServer);
+	Type -> Type
+    end.
+
 %%====================================================================
 %% gen_server callbacks
 %%====================================================================
@@ -661,7 +669,7 @@ transform_options({{s2s_host, Host}, Action}, Opts) ->
     ?WARNING_MSG("Option 's2s_host' is deprecated. "
                  "The option is still supported but it is better to "
                  "fix your config: use access rules instead.", []),
-    ACLName = jlib:binary_to_atom(
+    ACLName = misc:binary_to_atom(
                 iolist_to_binary(["s2s_access_", Host])),
     [{acl, ACLName, {server, Host}},
      {access, s2s, [{Action, ACLName}]},
@@ -739,7 +747,9 @@ opt_type(s2s_timeout) ->
     fun(I) when is_integer(I), I>=0 -> I;
        (infinity) -> infinity
     end;
+opt_type(s2s_queue_type) ->
+    fun(ram) -> ram; (file) -> file end;
 opt_type(_) ->
     [route_subdomains, s2s_access,  s2s_certfile,
      s2s_ciphers, s2s_dhfile, s2s_cafile, s2s_protocol_options,
-     s2s_tls_compression, s2s_use_starttls, s2s_timeout].
+     s2s_tls_compression, s2s_use_starttls, s2s_timeout, s2s_queue_type].

src/ejabberd_s2s_in.erl

@@ -182,7 +182,7 @@ handle_auth_success(RServer, Mech, _AuthModule,
 		      lserver := LServer} = State) ->
     ?INFO_MSG("(~s) Accepted inbound s2s ~s authentication ~s -> ~s (~s)",
 	      [SockMod:pp(Socket), Mech, RServer, LServer,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP))]),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP))]),
     State1 = case ejabberd_s2s:allow_host(ServerHost, RServer) of
 	       true ->
 		     AuthDomains1 = sets:add_element(RServer, AuthDomains),
@@ -200,7 +200,7 @@ handle_auth_failure(RServer, Mech, Reason,
 		      lserver := LServer} = State) ->
     ?INFO_MSG("(~s) Failed inbound s2s ~s authentication ~s -> ~s (~s): ~s",
 	      [SockMod:pp(Socket), Mech, RServer, LServer,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP)), Reason]),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP)), Reason]),
     ejabberd_hooks:run_fold(s2s_in_auth_result,
 			    ServerHost, State, [false, RServer]).
 

src/ejabberd_s2s_out.erl

@@ -145,14 +145,14 @@ process_closed(#{server := LServer, remote_server := RServer,
 		 on_route := send} = State,
 	       Reason) ->
     ?INFO_MSG("Closing outbound s2s connection ~s -> ~s: ~s",
-	      [LServer, RServer, xmpp_stream_out:format_error(Reason)]),
+	      [LServer, RServer, format_error(Reason)]),
     stop(State);
 process_closed(#{server := LServer, remote_server := RServer} = State,
 	       Reason) ->
     Delay = get_delay(),
     ?INFO_MSG("Failed to establish outbound s2s connection ~s -> ~s: ~s; "
 	      "bouncing for ~p seconds",
-	      [LServer, RServer, xmpp_stream_out:format_error(Reason), Delay]),
+	      [LServer, RServer, format_error(Reason), Delay]),
     State1 = State#{on_route => bounce},
     State2 = bounce_queue(State1),
     xmpp_stream_out:set_timeout(State2, timer:seconds(Delay)).
@@ -230,7 +230,7 @@ handle_auth_success(Mech, #{sockmod := SockMod,
 			    server := LServer} = State) ->
     ?INFO_MSG("(~s) Accepted outbound s2s ~s authentication ~s -> ~s (~s)",
 	      [SockMod:pp(Socket), Mech, LServer, RServer,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP))]),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP))]),
     ejabberd_hooks:run_fold(s2s_out_auth_result, ServerHost, State, [true]).
 
 handle_auth_failure(Mech, Reason,
@@ -241,7 +241,7 @@ handle_auth_failure(Mech, Reason,
 		      server := LServer} = State) ->
     ?INFO_MSG("(~s) Failed outbound s2s ~s authentication ~s -> ~s (~s): ~s",
 	      [SockMod:pp(Socket), Mech, LServer, RServer,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP)),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP)),
 	       xmpp_stream_out:format_error(Reason)]),
     ejabberd_hooks:run_fold(s2s_out_auth_result, ServerHost, State, [{false, Reason}]).
 
@@ -277,8 +277,14 @@ handle_timeout(#{on_route := Action} = State) ->
 
 init([#{server := LServer, remote_server := RServer} = State, Opts]) ->
     ServerHost = ejabberd_router:host_of_route(LServer),
+    QueueType = ejabberd_s2s:queue_type(LServer),
+    QueueLimit = case lists:keyfind(
+			max_queue, 1, ejabberd_config:fsm_limit_opts([])) of
+		     {_, N} -> N;
+		     false -> unlimited
+		 end,
     State1 = State#{on_route => queue,
-		    queue => queue:new(),
+		    queue => p1_queue:new(QueueType, QueueLimit),
 		    xmlns => ?NS_SERVER,
 		    lang => ?MYLANG,
 		    server_host => ServerHost,
@@ -300,7 +306,13 @@ handle_cast(Msg, #{server_host := ServerHost} = State) ->
 
 handle_info({route, Pkt}, #{queue := Q, on_route := Action} = State) ->
     case Action of
-	queue -> State#{queue => queue:in(Pkt, Q)};
+	queue ->
+	    try State#{queue => p1_queue:in(Pkt, Q)}
+	    catch error:full ->
+		    Q1 = p1_queue:set_limit(Q, unlimited),
+		    Q2 = p1_queue:in(Pkt, Q1),
+		    handle_stream_end(queue_full, State#{queue => Q2})
+	    end;
 	bounce -> bounce_packet(Pkt, State);
 	send -> set_idle_timeout(send(State, Pkt))
     end;
@@ -324,20 +336,18 @@ code_change(_OldVsn, State, _Extra) ->
 %%% Internal functions
 %%%===================================================================
 -spec resend_queue(state()) -> state().
-resend_queue(#{queue := Q} = State) ->
-    State1 = State#{queue => queue:new()},
-    jlib:queue_foldl(
+resend_queue(State) ->
+    queue_fold(
       fun(Pkt, AccState) ->
 	      send(AccState, Pkt)
-      end, State1, Q).
+      end, State).
 
 -spec bounce_queue(state()) -> state().
-bounce_queue(#{queue := Q} = State) ->
-    State1 = State#{queue => queue:new()},
-    jlib:queue_foldl(
+bounce_queue(State) ->
+    queue_fold(
       fun(Pkt, AccState) ->
 	      bounce_packet(Pkt, AccState)
-      end, State1, Q).
+      end, State).
 
 -spec bounce_message_queue(state()) -> state().
 bounce_message_queue(State) ->
@@ -359,10 +369,12 @@ bounce_packet(_, State) ->
 
 -spec mk_bounce_error(binary(), state()) -> stanza_error().
 mk_bounce_error(Lang, #{stop_reason := Why}) ->
-    Reason = xmpp_stream_out:format_error(Why),
+    Reason = format_error(Why),
     case Why of
 	internal_failure ->
-	    xmpp:err_internal_server_error();
+	    xmpp:err_internal_server_error(Reason, Lang);
+	queue_full ->
+	    xmpp:err_resource_constraint(Reason, Lang);
 	{dns, _} ->
 	    xmpp:err_remote_server_not_found(Reason, Lang);
 					     _ ->
@@ -387,6 +399,23 @@ set_idle_timeout(#{on_route := send, server := LServer} = State) ->
 set_idle_timeout(State) ->
     State.
 
+-spec queue_fold(fun((xmpp_element(), state()) -> state()), state()) -> state().
+queue_fold(F, #{queue := Q} = State) ->
+    case p1_queue:out(Q) of
+	{{value, Pkt}, Q1} ->
+	    State1 = F(Pkt, State#{queue => Q1}),
+	    queue_fold(F, State1);
+	{empty, Q1} ->
+	    State#{queue => Q1}
+    end.
+
+format_error(internal_failure) ->
+    <<"Internal server error">>;
+format_error(queue_full) ->
+    <<"Stream queue is overloaded">>;
+format_error(Reason) ->
+    xmpp_stream_out:format_error(Reason).
+
 transform_options(Opts) ->
     lists:foldl(fun transform_options/2, [], Opts).
 

src/ejabberd_service.erl

@@ -153,7 +153,7 @@ get_password_fun(#{remote_server := RemoteServer,
 		    ?INFO_MSG("(~s) Domain ~s is unconfigured for "
 			      "external component from ~s",
 			      [SockMod:pp(Socket), RemoteServer,
-			       ejabberd_config:may_hide_data(jlib:ip_to_list(IP))]),
+			       ejabberd_config:may_hide_data(misc:ip_to_list(IP))]),
 		    {false, undefined}
 	    end
     end.
@@ -165,7 +165,7 @@ handle_auth_success(_, Mech, _,
     ?INFO_MSG("(~s) Accepted external component ~s authentication "
 	      "for ~s from ~s",
 	      [SockMod:pp(Socket), Mech, RemoteServer,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP))]),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP))]),
     		    lists:foreach(
     		      fun (H) ->
     			      ejabberd_router:register_route(H, ?MYNAME),
@@ -180,7 +180,7 @@ handle_auth_failure(_, Mech, Reason,
     ?INFO_MSG("(~s) Failed external component ~s authentication "
 	      "for ~s from ~s: ~s",
 	      [SockMod:pp(Socket), Mech, RemoteServer,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP)),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP)),
 	       Reason]),
     State.
 

src/ejabberd_sm.erl

@@ -177,7 +177,8 @@ check_in_subscription(Acc, User, Server, _JID, _Type, _Reason) ->
 
 -spec bounce_offline_message({bounce, message()} | any()) -> any().
 
-bounce_offline_message({bounce, Packet} = Acc) ->
+bounce_offline_message({bounce, #message{type = T} = Packet} = Acc)
+    when T == chat; T == groupchat; T == normal ->
     Lang = xmpp:get_lang(Packet),
     Txt = <<"User session not found">>,
     Err = xmpp:err_service_unavailable(Txt, Lang),
@@ -393,7 +394,7 @@ c2s_handle_info(State, _) ->
 init([]) ->
     process_flag(trap_exit, true),
     lists:foreach(fun(Mod) -> Mod:init() end, get_sm_backends()),
-    ets:new(sm_iqtable, [named_table, public]),
+    ets:new(sm_iqtable, [named_table, public, {read_concurrency, true}]),
     ejabberd_hooks:add(host_up, ?MODULE, host_up, 50),
     ejabberd_hooks:add(host_down, ?MODULE, host_down, 60),
     lists:foreach(fun host_up/1, ?MYHOSTS),
@@ -553,7 +554,7 @@ do_route(#presence{to = #jid{lresource = <<"">>} = To} = Packet) ->
       end, get_user_present_resources(LUser, LServer));
 do_route(#message{to = #jid{lresource = <<"">>}, type = T} = Packet) ->
     ?DEBUG("processing message to bare JID:~n~s", [xmpp:pp(Packet)]),
-    if T == chat; T == headline; T == normal; T == groupchat ->
+    if T == chat; T == headline; T == normal ->
 	    route_message(Packet);
        true ->
 	    Lang = xmpp:get_lang(Packet),
@@ -572,11 +573,13 @@ do_route(Packet) ->
     case online(Mod:get_sessions(LUser, LServer, LResource)) of
 	[] ->
 	    case Packet of
-		#message{type = T} when T == chat; T == normal;
-					T == headline; T == groupchat ->
+		#message{type = T} when T == chat; T == normal ->
 		    route_message(Packet);
+		#message{type = T} when T == headline ->
+		    ?DEBUG("dropping headline to unavailable resource:~n~s",
+			   [xmpp:pp(Packet)]);
 		#presence{} ->
-		    ?DEBUG("dropping presence to unavalable resource:~n~s",
+		    ?DEBUG("dropping presence to unavailable resource:~n~s",
 			   [xmpp:pp(Packet)]);
 		_ ->
 		    Lang = xmpp:get_lang(Packet),

src/ejabberd_sm_redis.erl

@@ -50,19 +50,18 @@ set_session(Session) ->
     SIDKey = sid_to_key(Session#session.sid),
     ServKey = server_to_key(element(2, Session#session.us)),
     USSIDKey = us_sid_to_key(Session#session.us, Session#session.sid),
-    case ejabberd_redis:qp([["HSET", USKey, SIDKey, T],
-			    ["HSET", ServKey, USSIDKey, T]]) of
-	[{ok, _}, {ok, _}] ->
-	    ok;
-	Err ->
-	    ?ERROR_MSG("failed to set session for redis: ~p", [Err])
-    end.
+    ejabberd_redis:multi(
+      fun() ->
+	      ejabberd_redis:hset(USKey, SIDKey, T),
+	      ejabberd_redis:hset(ServKey, USSIDKey, T)
+      end),
+    ok.
 
 -spec delete_session(binary(), binary(), binary(), sid()) ->
 			    {ok, #session{}} | {error, notfound}.
 delete_session(LUser, LServer, _LResource, SID) ->
     USKey = us_to_key({LUser, LServer}),
-    case ejabberd_redis:q(["HGETALL", USKey]) of
+    case ejabberd_redis:hgetall(USKey) of
 	{ok, Vals} ->
 	    Ss = decode_session_list(Vals),
 	    case lists:keyfind(SID, #session.sid, Ss) of
@@ -72,12 +71,14 @@ delete_session(LUser, LServer, _LResource, SID) ->
 		    SIDKey = sid_to_key(SID),
 		    ServKey = server_to_key(element(2, Session#session.us)),
 		    USSIDKey = us_sid_to_key(Session#session.us, SID),
-		    ejabberd_redis:qp([["HDEL", USKey, SIDKey],
-				       ["HDEL", ServKey, USSIDKey]]),
+		    ejabberd_redis:multi(
+		      fun() ->
+			      ejabberd_redis:hdel(USKey, [SIDKey]),
+			      ejabberd_redis:hdel(ServKey, [USSIDKey])
+		      end),
 		    {ok, Session}
 	    end;
-	Err ->
-	    ?ERROR_MSG("failed to delete session from redis: ~p", [Err]),
+	{error, _} ->
 	    {error, notfound}
     end.
 
@@ -91,22 +92,20 @@ get_sessions() ->
 -spec get_sessions(binary()) -> [#session{}].
 get_sessions(LServer) ->
     ServKey = server_to_key(LServer),
-    case ejabberd_redis:q(["HGETALL", ServKey]) of
+    case ejabberd_redis:hgetall(ServKey) of
 	{ok, Vals} ->
 	    decode_session_list(Vals);
-	Err ->
-	    ?ERROR_MSG("failed to get sessions from redis: ~p", [Err]),
+	{error, _} ->
 	    []
     end.
 
 -spec get_sessions(binary(), binary()) -> [#session{}].
 get_sessions(LUser, LServer) ->
     USKey = us_to_key({LUser, LServer}),
-    case ejabberd_redis:q(["HGETALL", USKey]) of
-	{ok, Vals} when is_list(Vals) ->
+    case ejabberd_redis:hgetall(USKey) of
+	{ok, Vals} ->
 	    decode_session_list(Vals);
-	Err ->
-	    ?ERROR_MSG("failed to get sessions from redis: ~p", [Err]),
+	{error, _} ->
 	    []
     end.
 
@@ -114,12 +113,11 @@ get_sessions(LUser, LServer) ->
     [#session{}].
 get_sessions(LUser, LServer, LResource) ->
     USKey = us_to_key({LUser, LServer}),
-    case ejabberd_redis:q(["HGETALL", USKey]) of
-	{ok, Vals} when is_list(Vals) ->
+    case ejabberd_redis:hgetall(USKey) of
+	{ok, Vals} ->
 	    [S || S <- decode_session_list(Vals),
 		  element(3, S#session.usr) == LResource];
-	Err ->
-	    ?ERROR_MSG("failed to get sessions from redis: ~p", [Err]),
+	{error, _} ->
 	    []
     end.
 
@@ -141,52 +139,36 @@ us_sid_to_key(US, SID) ->
 sid_to_key(SID) ->
     term_to_binary(SID).
 
-decode_session_list([_, Val|T]) ->
-    [binary_to_term(Val)|decode_session_list(T)];
-decode_session_list([]) ->
-    [].
+decode_session_list(Vals) ->
+  [binary_to_term(Val) || {_, Val} <- Vals].
 
 clean_table() ->
     ?INFO_MSG("Cleaning Redis SM table...", []),
-    lists:foreach(
-      fun(LServer) ->
-	      ServKey = server_to_key(LServer),
-	      case ejabberd_redis:q(["HKEYS", ServKey]) of
-		  {ok, []} ->
-		      ok;
-		  {ok, Vals} ->
-		      Vals1 = lists:filter(
-				fun(USSIDKey) ->
-					{_, SID} = binary_to_term(USSIDKey),
-					node(element(2, SID)) == node()
-				end, Vals),
-				Q1 = case Vals1 of
-					[] -> [];
-					_ -> ["HDEL", ServKey | Vals1]
-				end,
-		      Q2 = lists:map(
-			     fun(USSIDKey) ->
-				     {US, SID} = binary_to_term(USSIDKey),
-				     USKey = us_to_key(US),
-				     SIDKey = sid_to_key(SID),
-				     ["HDEL", USKey, SIDKey]
-			     end, Vals1),
-		      Res = ejabberd_redis:qp(lists:delete([], [Q1|Q2])),
-		      case lists:filter(
-			     fun({ok, _}) -> false;
-				(_) -> true
-			     end, Res) of
-			  [] ->
-			      ok;
-			  Errs ->
-			      ?ERROR_MSG("failed to clean redis table for "
-					 "server ~s: ~p", [LServer, Errs])
-		      end;
-		  Err ->
-		      ?ERROR_MSG("failed to clean redis table for "
-				 "server ~s: ~p", [LServer, Err])
-	      end
-      end, ejabberd_sm:get_vh_by_backend(?MODULE)).
+    try
+	lists:foreach(
+	  fun(LServer) ->
+		  ServKey = server_to_key(LServer),
+		  {ok, Vals} = ejabberd_redis:hkeys(ServKey),
+		  {ok, _} =
+		      ejabberd_redis:multi(
+			fun() ->
+				lists:foreach(
+				  fun(USSIDKey) ->
+					  {US, SID} = binary_to_term(USSIDKey),
+					  if node(element(2, SID)) == node() ->
+						  USKey = us_to_key(US),
+						  SIDKey = sid_to_key(SID),
+						  ejabberd_redis:hdel(ServKey, [USSIDKey]),
+						  ejabberd_redis:hdel(USKey, [SIDKey]);
+					     true ->
+						  ok
+					  end
+				  end, Vals)
+			end)
+	  end, ejabberd_sm:get_vh_by_backend(?MODULE))
+    catch _:{badmatch, {error, _}} ->
+	    ?ERROR_MSG("failed to clean redis c2s sessions", [])
+    end.
 
 opt_type(redis_connect_timeout) ->
     fun (I) when is_integer(I), I > 0 -> I end;

src/ejabberd_sm_sql.erl

@@ -65,10 +65,10 @@ init() ->
 
 set_session(#session{sid = {Now, Pid}, usr = {U, LServer, R},
 		     priority = Priority, info = Info}) ->
-    InfoS = jlib:term_to_expr(Info),
+    InfoS = misc:term_to_expr(Info),
     PrioS = enc_priority(Priority),
     TS = now_to_timestamp(Now),
-    PidS = enc_pid(Pid),
+    PidS = misc:encode_pid(Pid),
     Node = erlang:atom_to_binary(node(Pid), latin1),
     case ?SQL_UPSERT(LServer, "sm",
                      ["!usec=%(TS)d",
@@ -98,7 +98,7 @@ delete_session(_LUser, LServer, _LResource, {Now, Pid}) ->
               ?SQL("delete from sm"
                    " where usec=%(TS)d and pid=%(PidS)s")),
 	    try {ok, row_to_session(LServer, Row)}
-	    catch _:{node_down, _} -> {error, notfound}
+	    catch _:{bad_node, _} -> {error, notfound}
 	    end;
 	{selected, []} ->
 	    {error, notfound};
@@ -122,7 +122,7 @@ get_sessions(LServer) ->
 	    lists:flatmap(
 	      fun(Row) ->
 		      try [row_to_session(LServer, Row)]
-		      catch _:{node_down, _} -> []
+		      catch _:{bad_node, _} -> []
 		      end
 	      end, Rows);
 	Err ->
@@ -140,7 +140,7 @@ get_sessions(LUser, LServer) ->
 	    lists:flatmap(
 	      fun(Row) ->
 		      try [row_to_session(LServer, Row)]
-		      catch _:{node_down, _} -> []
+		      catch _:{bad_node, _} -> []
 		      end
 	      end, Rows);
 	Err ->
@@ -158,7 +158,7 @@ get_sessions(LUser, LServer, LResource) ->
 	    lists:flatmap(
 	      fun(Row) ->
 		      try [row_to_session(LServer, Row)]
-		      catch _:{node_down, _} -> []
+		      catch _:{bad_node, _} -> []
 		      end
 	      end, Rows);
 	Err ->
@@ -194,37 +194,10 @@ enc_priority(Int) when is_integer(Int) ->
 
 row_to_session(LServer, {USec, PidS, NodeS, User, Resource, PrioS, InfoS}) ->
     Now = timestamp_to_now(USec),
-    Pid = dec_pid(PidS, NodeS),
+    Pid = misc:decode_pid(PidS, NodeS),
     Priority = dec_priority(PrioS),
     Info = ejabberd_sql:decode_term(InfoS),
     #session{sid = {Now, Pid}, us = {User, LServer},
 	     usr = {User, LServer, Resource},
 	     priority = Priority,
 	     info = Info}.
-
--spec enc_pid(pid()) -> binary().
-enc_pid(Pid) ->
-    list_to_binary(erlang:pid_to_list(Pid)).
-
--spec dec_pid(binary(), binary()) -> pid().
-dec_pid(PidBin, NodeBin) ->
-    PidStr = binary_to_list(PidBin),
-    Pid = erlang:list_to_pid(PidStr),
-    case erlang:binary_to_atom(NodeBin, latin1) of
-	Node when Node == node() ->
-	    Pid;
-	Node ->
-	    try set_node_id(PidStr, NodeBin)
-	    catch _:badarg ->
-		    erlang:error({node_down, Node})
-	    end
-    end.
-
--spec set_node_id(string(), binary()) -> pid().
-set_node_id(PidStr, NodeBin) ->
-    ExtPidStr = erlang:pid_to_list(
-		  binary_to_term(
-		    <<131,103,100,(size(NodeBin)):16,NodeBin/binary,0:72>>)),
-    [H|_] = string:tokens(ExtPidStr, "."),
-    [_|T] = string:tokens(PidStr, "."),
-    erlang:list_to_pid(string:join([H|T], ".")).

src/ejabberd_sql.erl

@@ -39,6 +39,8 @@
 	 sql_query_t/1,
 	 sql_transaction/2,
 	 sql_bloc/2,
+         abort/1,
+         restart/1,
          sql_query_to_iolist/1,
 	 escape/1,
          standard_escape/1,
@@ -75,8 +77,7 @@
 	 db_version = undefined              :: undefined | non_neg_integer(),
 	 start_interval = 0                  :: non_neg_integer(),
 	 host = <<"">>                       :: binary(),
-	 max_pending_requests_len            :: non_neg_integer(),
-	 pending_requests = {0, queue:new()} :: {non_neg_integer(), ?TQUEUE}}).
+	 pending_requests                    :: p1_queue:queue()}).
 
 -define(STATE_KEY, ejabberd_sql_state).
 
@@ -191,6 +192,12 @@ sql_query_t(Query) ->
       _ -> QRes
     end.
 
+abort(Reason) ->
+    exit(Reason).
+
+restart(Reason) ->
+    throw({aborted, Reason}).
+
 %% Escape character that will confuse an SQL engine
 escape(S) ->
 	<<  <<(sql_queries:escape(Char))/binary>> || <<Char>> <= S >>.
@@ -271,10 +278,16 @@ init([Host, StartInterval]) ->
     [DBType | _] = db_opts(Host),
     (?GEN_FSM):send_event(self(), connect),
     ejabberd_sql_sup:add_pid(Host, self()),
+    QueueType = case ejabberd_config:get_option(
+		       {sql_queue_type, Host}, opt_type(sql_queue_type)) of
+		    undefined ->
+			ejabberd_config:default_queue_type(Host);
+		    Type ->
+			Type
+		end,
     {ok, connecting,
      #state{db_type = DBType, host = Host,
-	    max_pending_requests_len = max_fsm_queue(),
-	    pending_requests = {0, queue:new()},
+	    pending_requests = p1_queue:new(QueueType, max_fsm_queue()),
 	    start_interval = StartInterval}}.
 
 connecting(connect, #state{host = Host} = State) ->
@@ -285,16 +298,23 @@ connecting(connect, #state{host = Host} = State) ->
 		   [mssql | Args] -> apply(fun odbc_connect/1, Args);
 		   [odbc | Args] -> apply(fun odbc_connect/1, Args)
 		 end,
-    {_, PendingRequests} = State#state.pending_requests,
     case ConnectRes of
         {ok, Ref} ->
             erlang:monitor(process, Ref),
-            lists:foreach(fun (Req) ->
-                                  (?GEN_FSM):send_event(self(), Req)
-                          end,
-                          queue:to_list(PendingRequests)),
+            lists:foreach(
+              fun({{?PREPARE_KEY, _} = Key, _}) ->
+                      erase(Key);
+                 (_) ->
+                      ok
+              end, get()),
+	    PendingRequests =
+		p1_queue:dropwhile(
+		  fun(Req) ->
+			  ?GEN_FSM:send_event(self(), Req),
+			  true
+		  end, State#state.pending_requests),
             State1 = State#state{db_ref = Ref,
-                                 pending_requests = {0, queue:new()}},
+                                 pending_requests = PendingRequests},
             State2 = get_db_version(State1),
             {next_state, session_established, State2};
       {error, Reason} ->
@@ -321,26 +341,20 @@ connecting({sql_cmd, Command, Timestamp} = Req, From,
 	   State) ->
     ?DEBUG("queuing pending request while connecting:~n\t~p",
 	   [Req]),
-    {Len, PendingRequests} = State#state.pending_requests,
-    NewPendingRequests = if Len <
-			      State#state.max_pending_requests_len ->
-				{Len + 1,
-				 queue:in({sql_cmd, Command, From, Timestamp},
-					  PendingRequests)};
-			    true ->
-				lists:foreach(fun ({sql_cmd, _, To,
-						    _Timestamp}) ->
-						      (?GEN_FSM):reply(To,
-								       {error,
-									<<"SQL connection failed">>})
-					      end,
-					      queue:to_list(PendingRequests)),
-				{1,
-				 queue:from_list([{sql_cmd, Command, From,
-						   Timestamp}])}
-			 end,
+    PendingRequests =
+	try p1_queue:in({sql_cmd, Command, From, Timestamp},
+			State#state.pending_requests)
+	catch error:full ->
+		Q = p1_queue:dropwhile(
+		      fun({sql_cmd, _, To, _Timestamp}) ->
+			      (?GEN_FSM):reply(
+				To, {error, <<"SQL connection failed">>}),
+			      true
+		      end, State#state.pending_requests),
+		p1_queue:in({sql_cmd, Command, From, Timestamp}, Q)
+	end,
     {next_state, connecting,
-     State#state{pending_requests = NewPendingRequests}};
+     State#state{pending_requests = PendingRequests}};
 connecting(Request, {Who, _Ref}, State) ->
     ?WARNING_MSG("unexpected call ~p from ~p in 'connecting'",
 		 [Request, Who]),
@@ -630,7 +644,7 @@ generic_sql_query_format(SQLQuery) ->
 
 generic_escape() ->
     #sql_escape{string = fun(X) -> <<"'", (escape(X))/binary, "'">> end,
-                integer = fun(X) -> jlib:i2l(X) end,
+                integer = fun(X) -> misc:i2l(X) end,
                 boolean = fun(true) -> <<"1">>;
                              (false) -> <<"0">>
                           end
@@ -647,7 +661,7 @@ sqlite_sql_query_format(SQLQuery) ->
 
 sqlite_escape() ->
     #sql_escape{string = fun(X) -> <<"'", (standard_escape(X))/binary, "'">> end,
-                integer = fun(X) -> jlib:i2l(X) end,
+                integer = fun(X) -> misc:i2l(X) end,
                 boolean = fun(true) -> <<"1">>;
                              (false) -> <<"0">>
                           end
@@ -671,7 +685,7 @@ pgsql_prepare(SQLQuery, State) ->
 
 pgsql_execute_escape() ->
     #sql_escape{string = fun(X) -> X end,
-                integer = fun(X) -> [jlib:i2l(X)] end,
+                integer = fun(X) -> [misc:i2l(X)] end,
                 boolean = fun(true) -> "1";
                              (false) -> "0"
                           end
@@ -1068,15 +1082,10 @@ odbcinst_config() ->
     filename:join(tmp_dir(), "odbcinst.ini").
 
 max_fsm_queue() ->
-    ejabberd_config:get_option(
-      max_fsm_queue,
-      fun(N) when is_integer(N), N > 0 -> N end).
+    proplists:get_value(max_queue, fsm_limit_opts(), unlimited).
 
 fsm_limit_opts() ->
-    case max_fsm_queue() of
-      N when is_integer(N) -> [{max_queue, N}];
-      _ -> []
-    end.
+    ejabberd_config:fsm_limit_opts([]).
 
 check_error({error, Why} = Err, #sql_query{} = Query) ->
     ?ERROR_MSG("SQL query '~s' at ~p failed: ~p",
@@ -1093,8 +1102,6 @@ check_error({error, Why} = Err, Query) ->
 check_error(Result, _Query) ->
     Result.
 
-opt_type(max_fsm_queue) ->
-    fun (N) when is_integer(N), N > 0 -> N end;
 opt_type(sql_database) -> fun iolist_to_binary/1;
 opt_type(sql_keepalive_interval) ->
     fun (I) when is_integer(I), I > 0 -> I end;
@@ -1114,8 +1121,10 @@ opt_type(sql_ssl) -> fun(B) when is_boolean(B) -> B end;
 opt_type(sql_ssl_verify) -> fun(B) when is_boolean(B) -> B end;
 opt_type(sql_ssl_certfile) -> fun iolist_to_binary/1;
 opt_type(sql_ssl_cafile) -> fun iolist_to_binary/1;
+opt_type(sql_queue_type) ->
+    fun(ram) -> ram; (file) -> file end;
 opt_type(_) ->
-    [max_fsm_queue, sql_database, sql_keepalive_interval,
+    [sql_database, sql_keepalive_interval,
      sql_password, sql_port, sql_server, sql_type,
      sql_username, sql_ssl, sql_ssl_verify, sql_ssl_cerfile,
-     sql_ssl_cafile].
+     sql_ssl_cafile, sql_queue_type].

src/ejabberd_sup.erl

@@ -115,8 +115,9 @@ init([]) ->
     RiakSupervisor = {ejabberd_riak_sup,
 		     {ejabberd_riak_sup, start_link, []},
 		      permanent, infinity, supervisor, [ejabberd_riak_sup]},
-    Redis = {ejabberd_redis, {ejabberd_redis, start_link, []},
-	     permanent, 5000, worker, [ejabberd_redis]},
+    RedisSupervisor = {ejabberd_redis_sup,
+		       {ejabberd_redis_sup, start_link, []},
+		       permanent, infinity, supervisor, [ejabberd_redis_sup]},
     Router = {ejabberd_router, {ejabberd_router, start_link, []},
 	      permanent, 5000, worker, [ejabberd_router]},
     RouterMulticast = {ejabberd_router_multicast,
@@ -168,12 +169,12 @@ init([]) ->
 	   BackendSupervisor,
 	   SQLSupervisor,
 	   RiakSupervisor,
-	   Redis,
+	   RedisSupervisor,
 	   Router,
 	   RouterMulticast,
 	   Local,
 	   SM,
-	   GenModSupervisor,
 	   ExtMod,
+	   GenModSupervisor,
 	   Auth,
 	   OAuth]}}.

src/ejabberd_system_monitor.erl

@@ -296,14 +296,14 @@ process_command1(From, To, Body) ->
     process_command2(str:tokens(Body, <<" ">>), From, To).
 
 process_command2([<<"kill">>, SNode, SPid], From, To) ->
-    Node = jlib:binary_to_atom(SNode),
+    Node = misc:binary_to_atom(SNode),
     remote_command(Node, [kill, SPid], From, To);
 process_command2([<<"showlh">>, SNode], From, To) ->
-    Node = jlib:binary_to_atom(SNode),
+    Node = misc:binary_to_atom(SNode),
     remote_command(Node, [showlh], From, To);
 process_command2([<<"setlh">>, SNode, NewValueString],
 		 From, To) ->
-    Node = jlib:binary_to_atom(SNode),
+    Node = misc:binary_to_atom(SNode),
     NewValue = binary_to_integer(NewValueString),
     remote_command(Node, [setlh, NewValue], From, To);
 process_command2([<<"help">>], From, To) ->

src/ejabberd_websocket.erl

@@ -152,7 +152,7 @@ handshake(#ws{headers = Headers} = State) ->
                             V ->
                                 [<<"Sec-Websocket-Protocol:">>, V, <<"\r\n">>]
                         end,
-    Hash = jlib:encode_base64(
+    Hash = misc:encode_base64(
              crypto:hash(sha, <<Key/binary, "258EAFA5-E914-47DA-95CA-C5AB0DC85B11">>)),
     {State, [<<"HTTP/1.1 101 Switching Protocols\r\n">>,
              <<"Upgrade: websocket\r\n">>,

src/eldap.erl

@@ -145,7 +145,7 @@
 %%% API
 %%%----------------------------------------------------------------------
 start_link(Name) ->
-    Reg_name = jlib:binary_to_atom(<<"eldap_",
+    Reg_name = misc:binary_to_atom(<<"eldap_",
 				       Name/binary>>),
     gen_fsm:start_link({local, Reg_name}, ?MODULE, [], []).
 
@@ -153,7 +153,7 @@ start_link(Name) ->
                  binary(), tlsopts()) -> any().
 
 start_link(Name, Hosts, Port, Rootdn, Passwd, Opts) ->
-    Reg_name = jlib:binary_to_atom(<<"eldap_",
+    Reg_name = misc:binary_to_atom(<<"eldap_",
 				       Name/binary>>),
     gen_fsm:start_link({local, Reg_name}, ?MODULE,
 		       [Hosts, Port, Rootdn, Passwd, Opts], []).
@@ -548,7 +548,7 @@ extensibleMatch_opts([], MRA) -> MRA.
 get_handle(Pid) when is_pid(Pid) -> Pid;
 get_handle(Atom) when is_atom(Atom) -> Atom;
 get_handle(Name) when is_binary(Name) ->
-    jlib:binary_to_atom(<<"eldap_",
+    misc:binary_to_atom(<<"eldap_",
 			    Name/binary>>).
 
 %%%----------------------------------------------------------------------

src/eldap_pool.erl

@@ -83,4 +83,4 @@ do_request(Name, {F, Args}) ->
     end.
 
 make_id(Name) ->
-    jlib:binary_to_atom(<<"eldap_pool_", Name/binary>>).
+    misc:binary_to_atom(<<"eldap_pool_", Name/binary>>).

src/ext_mod.erl

@@ -160,9 +160,9 @@ available() ->
                 lists:keystore(Key, 1, Acc, {Key, Val})
             end, Jungle, Standalone)).
 available(Module) when is_atom(Module) ->
-    available(jlib:atom_to_binary(Module));
+    available(misc:atom_to_binary(Module));
 available(Package) when is_binary(Package) ->
-    Available = [jlib:atom_to_binary(K) || K<-proplists:get_keys(available())],
+    Available = [misc:atom_to_binary(K) || K<-proplists:get_keys(available())],
     lists:member(Package, Available).
 
 available_command() ->
@@ -171,18 +171,18 @@ available_command() ->
 installed() ->
     modules_spec(modules_dir(), "*").
 installed(Module) when is_atom(Module) ->
-    installed(jlib:atom_to_binary(Module));
+    installed(misc:atom_to_binary(Module));
 installed(Package) when is_binary(Package) ->
-    Installed = [jlib:atom_to_binary(K) || K<-proplists:get_keys(installed())],
+    Installed = [misc:atom_to_binary(K) || K<-proplists:get_keys(installed())],
     lists:member(Package, Installed).
 
 installed_command() ->
     [short_spec(Item) || Item <- installed()].
 
 install(Module) when is_atom(Module) ->
-    install(jlib:atom_to_binary(Module));
+    install(misc:atom_to_binary(Module));
 install(Package) when is_binary(Package) ->
-    Spec = [S || {Mod, S} <- available(), jlib:atom_to_binary(Mod)==Package],
+    Spec = [S || {Mod, S} <- available(), misc:atom_to_binary(Mod)==Package],
     case {Spec, installed(Package), is_contrib_allowed()} of
         {_, _, false} ->
             {error, not_allowed};
@@ -191,7 +191,7 @@ install(Package) when is_binary(Package) ->
         {_, true, _} ->
             {error, conflict};
         {[Attrs], _, _} ->
-            Module = jlib:binary_to_atom(Package),
+            Module = misc:binary_to_atom(Package),
             case compile_and_install(Module, Attrs) of
                 ok ->
                     code:add_patha(module_ebin_dir(Module)),
@@ -207,11 +207,11 @@ install(Package) when is_binary(Package) ->
     end.
 
 uninstall(Module) when is_atom(Module) ->
-    uninstall(jlib:atom_to_binary(Module));
+    uninstall(misc:atom_to_binary(Module));
 uninstall(Package) when is_binary(Package) ->
     case installed(Package) of
         true ->
-            Module = jlib:binary_to_atom(Package),
+            Module = misc:binary_to_atom(Package),
             case erlang:function_exported(Module, pre_uninstall, 0) of
                 true -> Module:pre_uninstall();
                 _ -> ok
@@ -230,7 +230,7 @@ uninstall(Package) when is_binary(Package) ->
 upgrade() ->
     [{Package, upgrade(Package)} || {Package, _Spec} <- installed()].
 upgrade(Module) when is_atom(Module) ->
-    upgrade(jlib:atom_to_binary(Module));
+    upgrade(misc:atom_to_binary(Module));
 upgrade(Package) when is_binary(Package) ->
     uninstall(Package),
     install(Package).
@@ -240,7 +240,7 @@ add_sources(Path) when is_list(Path) ->
 add_sources(_, "") ->
     {error, no_url};
 add_sources(Module, Path) when is_atom(Module), is_list(Path) ->
-    add_sources(jlib:atom_to_binary(Module), Path);
+    add_sources(misc:atom_to_binary(Module), Path);
 add_sources(Package, Path) when is_binary(Package), is_list(Path) ->
     DestDir = sources_dir(),
     RepDir = filename:join(DestDir, module_name(Path)),
@@ -261,18 +261,18 @@ add_sources(Package, Path) when is_binary(Package), is_list(Path) ->
     end.
 
 del_sources(Module) when is_atom(Module) ->
-    del_sources(jlib:atom_to_binary(Module));
+    del_sources(misc:atom_to_binary(Module));
 del_sources(Package) when is_binary(Package) ->
     case uninstall(Package) of
         ok ->
-            SrcDir = module_src_dir(jlib:binary_to_atom(Package)),
+            SrcDir = module_src_dir(misc:binary_to_atom(Package)),
             delete_path(SrcDir);
         Error ->
             Error
     end.
 
 check(Module) when is_atom(Module) ->
-    check(jlib:atom_to_binary(Module));
+    check(misc:atom_to_binary(Module));
 check(Package) when is_binary(Package) ->
     case {available(Package), installed(Package)} of
         {false, _} ->
@@ -281,11 +281,11 @@ check(Package) when is_binary(Package) ->
             Status = install(Package),
             uninstall(Package),
             case Status of
-                ok -> check_sources(jlib:binary_to_atom(Package));
+                ok -> check_sources(misc:binary_to_atom(Package));
                 Error -> Error
             end;
         _ ->
-            check_sources(jlib:binary_to_atom(Package))
+            check_sources(misc:binary_to_atom(Package))
     end.
 
 %% -- archives and variables functions
@@ -420,7 +420,7 @@ module_name(Id) ->
     filename:basename(filename:rootname(Id)).
 
 module(Id) ->
-    jlib:binary_to_atom(iolist_to_binary(module_name(Id))).
+    misc:binary_to_atom(iolist_to_binary(module_name(Id))).
 
 module_spec(Spec) ->
     [{path, filename:dirname(Spec)}
@@ -541,8 +541,21 @@ compile_result(Results) ->
 
 compile_options() ->
     [verbose, report_errors, report_warnings]
-    ++ [{i, filename:join(code:lib_dir(App), "include")}
-        || App <- [fast_xml, xmpp, ejabberd]].
+    ++ [{i, filename:join(app_dir(App), "include")}
+        || App <- [fast_xml, xmpp, p1_utils, ejabberd]].
+
+app_dir(App) ->
+    case code:lib_dir(App) of
+        {error, bad_name} ->
+            case code:which(App) of
+                Beam when is_list(Beam) ->
+                    filename:dirname(filename:dirname(Beam));
+                _ ->
+                    "."
+            end;
+        Dir ->
+            Dir
+    end.
 
 compile_erlang_file(Dest, File) ->
     compile_erlang_file(Dest, File, compile_options()).

src/gen_mod.erl

@@ -85,7 +85,8 @@ init([]) ->
     ejabberd_hooks:add(host_down, ?MODULE, stop_modules, 70),
     ets:new(ejabberd_modules,
 	    [named_table, public,
-	     {keypos, #ejabberd_module.module_host}]),
+	     {keypos, #ejabberd_module.module_host},
+	     {read_concurrency, true}]),
     {ok, {{one_for_one, 10, 1}, []}}.
 
 -spec start_child(module(), binary() | global, opts()) -> ok | {error, any()}.

src/gen_pubsub_node.erl

@@ -183,6 +183,9 @@
 -callback get_items(nodeIdx(), jid(), undefined | rsm_set()) ->
     {result, {[pubsubItem()], undefined | rsm_set()}}.
 
+-callback get_last_items(nodeIdx(), jid(), undefined | rsm_set()) ->
+    {result, {[pubsubItem()], undefined | rsm_set()}}.
+
 -callback get_item(NodeIdx :: nodeIdx(),
 	ItemId :: itemId(),
 	JID :: jid(),

src/jlib.erl

@@ -35,12 +35,12 @@
                            binary_to_integer/1,
                            integer_to_binary/1]}).
 
+%% The following functions are deprected: use functions from aux.erl
 -export([tolower/1, term_to_base64/1, base64_to_term/1,
 	 decode_base64/1, encode_base64/1, ip_to_list/1,
-	 hex_to_bin/1, hex_to_base64/1,
+	 hex_to_bin/1, hex_to_base64/1, expand_keyword/3,
 	 atom_to_binary/1, binary_to_atom/1, tuple_to_binary/1,
-	 l2i/1, i2l/1, i2l/2, expr_to_term/1, term_to_expr/1,
-	 queue_drop_while/2, queue_foldl/3, queue_foldr/3, queue_foreach/2]).
+	 l2i/1, i2l/1, i2l/2, expr_to_term/1, term_to_expr/1]).
 
 %% The following functions are used by gen_iq_handler.erl for providing backward
 %% compatibility and must not be used in other parts of the code
@@ -112,7 +112,24 @@
 	     {binary_to_integer, 1},
 	     {binary_to_integer, 2},
 	     {integer_to_binary, 1},
-	     {integer_to_binary, 2}]).
+	     {integer_to_binary, 2},
+	     {tolower, 1},
+	     {term_to_base64, 1},
+	     {base64_to_term, 1},
+	     {decode_base64, 1},
+	     {encode_base64, 1},
+	     {ip_to_list, 1},
+	     {hex_to_bin, 1},
+	     {hex_to_base64, 1},
+	     {expand_keyword, 3},
+	     {atom_to_binary, 1},
+	     {binary_to_atom, 1},
+	     {tuple_to_binary, 1},
+	     {l2i, 1},
+	     {i2l, 1},
+	     {i2l, 2},
+	     {expr_to_term, 1},
+	     {term_to_expr, 1}]).
 
 -include("ejabberd.hrl").
 -include("jlib.hrl").
@@ -935,6 +952,12 @@ hex_to_bin([H1, H2 | T], Acc) ->
 
 hex_to_base64(Hex) -> encode_base64(hex_to_bin(Hex)).
 
+-spec expand_keyword(binary(), binary(), binary()) -> binary().
+
+expand_keyword(Keyword, Input, Replacement) ->
+    Parts = binary:split(Input, Keyword, [global]),
+    str:join(Parts, Replacement).
+
 binary_to_atom(Bin) ->
     erlang:binary_to_atom(Bin, utf8).
 
@@ -978,48 +1001,3 @@ i2l(L, N) when is_binary(L) ->
       C when C > N -> L;
       _ -> i2l(<<$0, L/binary>>, N)
     end.
-
--spec queue_drop_while(fun((term()) -> boolean()), ?TQUEUE) -> ?TQUEUE.
-
-queue_drop_while(F, Q) ->
-    case queue:peek(Q) of
-      {value, Item} ->
-	  case F(Item) of
-	    true ->
-		queue_drop_while(F, queue:drop(Q));
-	    _ ->
-		Q
-	  end;
-      empty ->
-	  Q
-    end.
-
--spec queue_foldl(fun((term(), T) -> T), T, ?TQUEUE) -> T.
-queue_foldl(F, Acc, Q) ->
-    case queue:out(Q) of
-	{{value, Item}, Q1} ->
-	    Acc1 = F(Item, Acc),
-	    queue_foldl(F, Acc1, Q1);
-	{empty, _} ->
-	    Acc
-    end.
-
--spec queue_foldr(fun((term(), T) -> T), T, ?TQUEUE) -> T.
-queue_foldr(F, Acc, Q) ->
-    case queue:out_r(Q) of
-	{{value, Item}, Q1} ->
-	    Acc1 = F(Item, Acc),
-	    queue_foldr(F, Acc1, Q1);
-	{empty, _} ->
-	    Acc
-    end.
-
--spec queue_foreach(fun((_) -> _), ?TQUEUE) -> ok.
-queue_foreach(F, Q) ->
-    case queue:out(Q) of
-	{{value, Item}, Q1} ->
-	    F(Item),
-	    queue_foreach(F, Q1);
-	{empty, _} ->
-	    ok
-    end.

src/misc.erl

@@ -0,0 +1,226 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% @doc
+%%%   This is the place for some unsorted auxiliary functions
+%%%   Some functions from jlib.erl are moved here
+%%%   Mild rubbish heap is accepted ;)
+%%% @end
+%%% Created : 30 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2017   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(misc).
+
+%% API
+-export([tolower/1, term_to_base64/1, base64_to_term/1,
+	 decode_base64/1, encode_base64/1, ip_to_list/1,
+	 hex_to_bin/1, hex_to_base64/1, expand_keyword/3,
+	 atom_to_binary/1, binary_to_atom/1, tuple_to_binary/1,
+	 l2i/1, i2l/1, i2l/2, expr_to_term/1, term_to_expr/1,
+	 encode_pid/1, decode_pid/2]).
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+-spec tolower(binary()) -> binary().
+tolower(B) ->
+    iolist_to_binary(tolower_s(binary_to_list(B))).
+
+tolower_s([C | Cs]) ->
+    if C >= $A, C =< $Z -> [C + 32 | tolower_s(Cs)];
+       true -> [C | tolower_s(Cs)]
+    end;
+tolower_s([]) -> [].
+
+-spec term_to_base64(term()) -> binary().
+term_to_base64(Term) ->
+    encode_base64(term_to_binary(Term)).
+
+-spec base64_to_term(binary()) -> {term, term()} | error.
+base64_to_term(Base64) ->
+    case catch binary_to_term(decode_base64(Base64), [safe]) of
+      {'EXIT', _} ->
+	  error;
+      Term ->
+	  {term, Term}
+    end.
+
+-spec decode_base64(binary()) -> binary().
+decode_base64(S) ->
+    case catch binary:last(S) of
+      C when C == $\n; C == $\s ->
+	  decode_base64(binary:part(S, 0, byte_size(S) - 1));
+      _ ->
+	  decode_base64_bin(S, <<>>)
+    end.
+
+take_without_spaces(Bin, Count) ->
+    take_without_spaces(Bin, Count, <<>>).
+
+take_without_spaces(Bin, 0, Acc) ->
+    {Acc, Bin};
+take_without_spaces(<<>>, _, Acc) ->
+    {Acc, <<>>};
+take_without_spaces(<<$\s, Tail/binary>>, Count, Acc) ->
+    take_without_spaces(Tail, Count, Acc);
+take_without_spaces(<<$\t, Tail/binary>>, Count, Acc) ->
+    take_without_spaces(Tail, Count, Acc);
+take_without_spaces(<<$\n, Tail/binary>>, Count, Acc) ->
+    take_without_spaces(Tail, Count, Acc);
+take_without_spaces(<<$\r, Tail/binary>>, Count, Acc) ->
+    take_without_spaces(Tail, Count, Acc);
+take_without_spaces(<<Char:8, Tail/binary>>, Count, Acc) ->
+    take_without_spaces(Tail, Count-1, <<Acc/binary, Char:8>>).
+
+decode_base64_bin(<<>>, Acc) ->
+    Acc;
+decode_base64_bin(Bin, Acc) ->
+    case take_without_spaces(Bin, 4) of
+        {<<A, B, $=, $=>>, _} ->
+            <<Acc/binary, (d(A)):6, (d(B) bsr 4):2>>;
+        {<<A, B, C, $=>>, _} ->
+            <<Acc/binary, (d(A)):6, (d(B)):6, (d(C) bsr 2):4>>;
+        {<<A, B, C, D>>, Tail} ->
+            Acc2 = <<Acc/binary, (d(A)):6, (d(B)):6, (d(C)):6, (d(D)):6>>,
+            decode_base64_bin(Tail, Acc2);
+        _ ->
+            <<"">>
+    end.
+
+d(X) when X >= $A, X =< $Z -> X - 65;
+d(X) when X >= $a, X =< $z -> X - 71;
+d(X) when X >= $0, X =< $9 -> X + 4;
+d($+) -> 62;
+d($/) -> 63;
+d(_) -> 63.
+
+
+%% Convert Erlang inet IP to list
+-spec encode_base64(binary()) -> binary().
+encode_base64(Data) ->
+    encode_base64_bin(Data, <<>>).
+
+encode_base64_bin(<<A:6, B:6, C:6, D:6, Tail/binary>>, Acc) ->
+    encode_base64_bin(Tail, <<Acc/binary, (e(A)):8, (e(B)):8, (e(C)):8, (e(D)):8>>);
+encode_base64_bin(<<A:6, B:6, C:4>>, Acc) ->
+    <<Acc/binary, (e(A)):8, (e(B)):8, (e(C bsl 2)):8, $=>>;
+encode_base64_bin(<<A:6, B:2>>, Acc) ->
+    <<Acc/binary, (e(A)):8, (e(B bsl 4)):8, $=, $=>>;
+encode_base64_bin(<<>>, Acc) ->
+    Acc.
+
+e(X) when X >= 0, X < 26 -> X + 65;
+e(X) when X > 25, X < 52 -> X + 71;
+e(X) when X > 51, X < 62 -> X - 4;
+e(62) -> $+;
+e(63) -> $/;
+e(X) -> exit({bad_encode_base64_token, X}).
+
+-spec ip_to_list(inet:ip_address() | undefined |
+                 {inet:ip_address(), inet:port_number()}) -> binary().
+
+ip_to_list({IP, _Port}) ->
+    ip_to_list(IP);
+%% This function clause could use inet_parse too:
+ip_to_list(undefined) ->
+    <<"unknown">>;
+ip_to_list(IP) ->
+    list_to_binary(inet_parse:ntoa(IP)).
+
+-spec hex_to_bin(binary()) -> binary().
+hex_to_bin(Hex) ->
+    hex_to_bin(binary_to_list(Hex), []).
+
+-spec hex_to_bin(list(), list()) -> binary().
+hex_to_bin([], Acc) ->
+    list_to_binary(lists:reverse(Acc));
+hex_to_bin([H1, H2 | T], Acc) ->
+    {ok, [V], []} = io_lib:fread("~16u", [H1, H2]),
+    hex_to_bin(T, [V | Acc]).
+
+-spec hex_to_base64(binary()) -> binary().
+hex_to_base64(Hex) ->
+    encode_base64(hex_to_bin(Hex)).
+
+-spec expand_keyword(binary(), binary(), binary()) -> binary().
+expand_keyword(Keyword, Input, Replacement) ->
+    Parts = binary:split(Input, Keyword, [global]),
+    str:join(Parts, Replacement).
+
+binary_to_atom(Bin) ->
+    erlang:binary_to_atom(Bin, utf8).
+
+tuple_to_binary(T) ->
+    iolist_to_binary(tuple_to_list(T)).
+
+atom_to_binary(A) ->
+    erlang:atom_to_binary(A, utf8).
+
+expr_to_term(Expr) ->
+    Str = binary_to_list(<<Expr/binary, ".">>),
+    {ok, Tokens, _} = erl_scan:string(Str),
+    {ok, Term} = erl_parse:parse_term(Tokens),
+    Term.
+
+term_to_expr(Term) ->
+    list_to_binary(io_lib:print(Term)).
+
+l2i(I) when is_integer(I) -> I;
+l2i(L) when is_binary(L) -> binary_to_integer(L).
+
+i2l(I) when is_integer(I) -> integer_to_binary(I);
+i2l(L) when is_binary(L) -> L.
+
+i2l(I, N) when is_integer(I) -> i2l(i2l(I), N);
+i2l(L, N) when is_binary(L) ->
+    case str:len(L) of
+      N -> L;
+      C when C > N -> L;
+      _ -> i2l(<<$0, L/binary>>, N)
+    end.
+
+-spec encode_pid(pid()) -> binary().
+encode_pid(Pid) ->
+    list_to_binary(erlang:pid_to_list(Pid)).
+
+-spec decode_pid(binary(), binary()) -> pid().
+decode_pid(PidBin, NodeBin) ->
+    PidStr = binary_to_list(PidBin),
+    Pid = erlang:list_to_pid(PidStr),
+    case erlang:binary_to_atom(NodeBin, latin1) of
+	Node when Node == node() ->
+	    Pid;
+	Node ->
+	    try set_node_id(PidStr, NodeBin)
+	    catch _:badarg ->
+		    erlang:error({bad_node, Node})
+	    end
+    end.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+-spec set_node_id(string(), binary()) -> pid().
+set_node_id(PidStr, NodeBin) ->
+    ExtPidStr = erlang:pid_to_list(
+		  binary_to_term(
+		    <<131,103,100,(size(NodeBin)):16,NodeBin/binary,0:72>>)),
+    [H|_] = string:tokens(ExtPidStr, "."),
+    [_|T] = string:tokens(PidStr, "."),
+    erlang:list_to_pid(string:join([H|T], ".")).

src/mod_admin_extra.erl

@@ -673,7 +673,7 @@ get_cookie() ->
     atom_to_list(erlang:get_cookie()).
 
 restart_module(Host, Module) when is_binary(Module) ->
-    restart_module(Host, jlib:binary_to_atom(Module));
+    restart_module(Host, misc:binary_to_atom(Module));
 restart_module(Host, Module) when is_atom(Module) ->
     List = gen_mod:loaded_modules_with_opts(Host),
     case proplists:get_value(Module, List) of
@@ -1031,8 +1031,8 @@ set_presence(User, Host, Resource, Type, Show, Status, Priority0) ->
 	    From = jid:make(User, Host, Resource),
 	    To = jid:make(User, Host),
 	    Presence = #presence{from = From, to = To,
-				 type = jlib:binary_to_atom(Type),
-				 show = jlib:binary_to_atom(Show),
+				 type = misc:binary_to_atom(Type),
+				 show = misc:binary_to_atom(Show),
 				 status = xmpp:mk_text(Status),
 				 priority = Priority},
 	    Pid ! {route, Presence},
@@ -1317,7 +1317,7 @@ build_roster_item(U, S, {add, Nick, Subs, Group}) ->
     Groups = binary:split(Group,<<";">>, [global]),
     #roster_item{jid = jid:make(U, S),
 		 name = Nick,
-		 subscription = jlib:binary_to_atom(Subs),
+		 subscription = misc:binary_to_atom(Subs),
 		 groups = Groups};
 build_roster_item(U, S, remove) ->
     #roster_item{jid = jid:make(U, S), subscription = remove}.
@@ -1410,7 +1410,7 @@ srg_get_info(Group, Host) ->
 	Os when is_list(Os) -> Os;
 	error -> []
     end,
-    [{jlib:atom_to_binary(Title), btl(Value)} || {Title, Value} <- Opts].
+    [{misc:atom_to_binary(Title), btl(Value)} || {Title, Value} <- Opts].
 
 btl([]) -> [];
 btl([B|L]) -> [btl(B)|btl(L)];
@@ -1443,7 +1443,7 @@ send_message(Type, From, To, Subject, Body) ->
     ejabberd_router:route(xmpp:set_from_to(Packet, FromJID, ToJID)).
 
 build_packet(Type, Subject, Body) ->
-    #message{type = jlib:binary_to_atom(Type),
+    #message{type = misc:binary_to_atom(Type),
 	     body = xmpp:mk_text(Body),
 	     subject = xmpp:mk_text(Subject)}.
 

src/mod_bosh.erl

@@ -158,9 +158,11 @@ mod_opt_type(prebind) ->
     fun (B) when is_boolean(B) -> B end;
 mod_opt_type(ram_db_type) ->
     fun(T) -> ejabberd_config:v_db(?MODULE, T) end;
+mod_opt_type(queue_type) ->
+    fun(ram) -> ram; (file) -> file end;
 mod_opt_type(_) ->
-    [json, max_concat, max_inactivity, max_pause, prebind, ram_db_type].
-
+    [json, max_concat, max_inactivity, max_pause, prebind, ram_db_type,
+     queue_type].
 
 %%%----------------------------------------------------------------------
 %%% Help Web Page

src/mod_bosh_redis.erl

@@ -0,0 +1,72 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% @copyright (C) 2017, Evgeny Khramtsov
+%%% @doc
+%%%
+%%% @end
+%%% Created : 28 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%-------------------------------------------------------------------
+-module(mod_bosh_redis).
+-behaviour(mod_bosh).
+
+%% API
+-export([init/0, open_session/2, close_session/1, find_session/1]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+
+-define(BOSH_KEY, "ejabberd:bosh").
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+init() ->
+    clean_table().
+
+open_session(SID, Pid) ->
+    PidBin = term_to_binary(Pid),
+    case ejabberd_redis:hset(?BOSH_KEY, SID, PidBin) of
+	{ok, _} ->
+	    ok;
+	{error, _} ->
+	    {error, db_failure}
+    end.
+
+close_session(SID) ->
+    ejabberd_redis:hdel(?BOSH_KEY, [SID]),
+    ok.
+
+find_session(SID) ->
+    case ejabberd_redis:hget(?BOSH_KEY, SID) of
+	{ok, Pid} when is_binary(Pid) ->
+	    try
+		{ok, binary_to_term(Pid)}
+	    catch _:badarg ->
+		    ?ERROR_MSG("malformed data in redis (key = '~s'): ~p",
+			       [SID, Pid]),
+		    error
+	    end;
+	_ ->
+	    error
+    end.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+clean_table() ->
+    ?INFO_MSG("Cleaning Redis BOSH sessions...", []),
+    case ejabberd_redis:hgetall(?BOSH_KEY) of
+	{ok, Vals} ->
+	    ejabberd_redis:multi(
+	      fun() ->
+		      lists:foreach(
+			fun({SID, Pid}) when node(Pid) == node() ->
+				ejabberd_redis:hdel(?BOSH_KEY, [SID]);
+			   (_) ->
+				ok
+			end, Vals)
+	      end),
+	    ok;
+	{error, _} ->
+	    ?ERROR_MSG("failed to clean bosh sessions in redis", [])
+    end.

src/mod_bosh_sql.erl

@@ -0,0 +1,72 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% @copyright (C) 2017, Evgeny Khramtsov
+%%% @doc
+%%%
+%%% @end
+%%% Created : 28 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%-------------------------------------------------------------------
+-module(mod_bosh_sql).
+-behaviour(mod_bosh).
+
+-compile([{parse_transform, ejabberd_sql_pt}]).
+
+%% API
+-export([init/0, open_session/2, close_session/1, find_session/1]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+-include("ejabberd_sql_pt.hrl").
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+init() ->
+    Node = erlang:atom_to_binary(node(), latin1),
+    ?INFO_MSG("Cleaning SQL 'bosh' table...", []),
+    case ejabberd_sql:sql_query(
+	   ?MYNAME, ?SQL("delete from bosh where node=%(Node)s")) of
+	{updated, _} ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to clean 'route' table: ~p", [Err]),
+	    Err
+    end.
+
+open_session(SID, Pid) ->
+    PidS = misc:encode_pid(Pid),
+    Node = erlang:atom_to_binary(node(Pid), latin1),
+    case ?SQL_UPSERT(?MYNAME, "bosh",
+		     ["!sid=%(SID)s",
+		      "node=%(Node)s",
+		      "pid=%(PidS)s"]) of
+	ok ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to update 'bosh' table: ~p", [Err]),
+	    {error, Err}
+    end.
+
+close_session(SID) ->
+    %% TODO: report errors
+    ejabberd_sql:sql_query(
+      ?MYNAME, ?SQL("delete from bosh where sid=%(SID)s")).
+
+find_session(SID) ->
+    case ejabberd_sql:sql_query(
+	   ?MYNAME,
+	   ?SQL("select @(pid)s, @(node)s from bosh where sid=%(SID)s")) of
+	{selected, [{Pid, Node}]} ->
+	    try	{ok, misc:decode_pid(Pid, Node)}
+	    catch _:{bad_node, _} -> error
+	    end;
+	{selected, []} ->
+	    error;
+	Err ->
+	    ?ERROR_MSG("failed to select 'bosh' table: ~p", [Err]),
+	    error
+    end.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================

src/mod_caps.erl

@@ -92,12 +92,12 @@ get_features(Host, #caps{node = Node, version = Version,
 
 -spec list_features(ejabberd_c2s:state()) -> [{ljid(), caps()}].
 list_features(C2SState) ->
-    Rs = maps:get(caps_features, C2SState, gb_trees:empty()),
+    Rs = maps:get(caps_resources, C2SState, gb_trees:empty()),
     gb_trees:to_list(Rs).
 
 -spec get_user_caps(jid(), ejabberd_c2s:state()) -> {ok, caps()} | error.
 get_user_caps(JID, C2SState) ->
-    Rs = maps:get(caps_features, C2SState, gb_trees:empty()),
+    Rs = maps:get(caps_resources, C2SState, gb_trees:empty()),
     LJID = jid:tolower(JID),
     case gb_trees:lookup(LJID, Rs) of
 	{value, Caps} ->
@@ -209,7 +209,7 @@ c2s_presence_in(C2SState,
     Delete = (Type == unavailable) or (Type == error),
     if Insert or Delete ->
 	   LFrom = jid:tolower(From),
-	    Rs = maps:get(caps_resources, C2SState, gb_trees:empty()),
+	   Rs = maps:get(caps_resources, C2SState, gb_trees:empty()),
 	   Caps = read_caps(Presence),
 	   NewRs = case Caps of
 		     nothing when Insert == true -> Rs;
@@ -420,7 +420,7 @@ make_my_disco_hash(Host) ->
 make_disco_hash(DiscoInfo, Algo) ->
     Concat = list_to_binary([concat_identities(DiscoInfo),
                              concat_features(DiscoInfo), concat_info(DiscoInfo)]),
-    jlib:encode_base64(case Algo of
+    misc:encode_base64(case Algo of
                            md5 -> erlang:md5(Concat);
                            sha -> crypto:hash(sha, Concat);
                            sha224 -> crypto:hash(sha224, Concat);

src/mod_carboncopy.erl

@@ -58,7 +58,7 @@ is_carbon_copy(_) ->
 start(Host, Opts) ->
     IQDisc = gen_mod:get_opt(iqdisc, Opts,fun gen_iq_handler:check_type/1, one_queue),
     ejabberd_hooks:add(disco_local_features, Host, ?MODULE, disco_features, 50),
-    Mod = gen_mod:db_mod(Host, ?MODULE),
+    Mod = gen_mod:ram_db_mod(Host, ?MODULE),
     Mod:init(Host, Opts),
     ejabberd_hooks:add(unset_presence_hook,Host, ?MODULE, remove_connection, 10),
     %% why priority 89: to define clearly that we must run BEFORE mod_logdb hook (90)
@@ -75,8 +75,8 @@ stop(Host) ->
     ejabberd_hooks:delete(unset_presence_hook,Host, ?MODULE, remove_connection, 10).
 
 reload(Host, NewOpts, OldOpts) ->
-    NewMod = gen_mod:db_mod(Host, NewOpts, ?MODULE),
-    OldMod = gen_mod:db_mod(Host, OldOpts, ?MODULE),
+    NewMod = gen_mod:ram_db_mod(Host, NewOpts, ?MODULE),
+    OldMod = gen_mod:ram_db_mod(Host, OldOpts, ?MODULE),
     if NewMod /= OldMod ->
 	    NewMod:init(Host, NewOpts);
        true ->
@@ -246,13 +246,13 @@ build_forward_packet(JID, #message{type = T} = Msg, Sender, Dest, Direction) ->
 -spec enable(binary(), binary(), binary(), binary()) -> ok | {error, any()}.
 enable(Host, U, R, CC)->
     ?DEBUG("enabling for ~p", [U]),
-    Mod = gen_mod:db_mod(Host, ?MODULE),
+    Mod = gen_mod:ram_db_mod(Host, ?MODULE),
     Mod:enable(U, Host, R, CC).
 
 -spec disable(binary(), binary(), binary()) -> ok | {error, any()}.
 disable(Host, U, R)->
     ?DEBUG("disabling for ~p", [U]),
-    Mod = gen_mod:db_mod(Host, ?MODULE),
+    Mod = gen_mod:ram_db_mod(Host, ?MODULE),
     Mod:disable(U, Host, R).
 
 -spec complete_packet(jid(), message(), direction()) -> message().
@@ -279,12 +279,12 @@ is_muc_pm(_To, Packet) ->
 -spec list(binary(), binary()) -> [{binary(), binary()}].
 %% list {resource, cc_version} with carbons enabled for given user and host
 list(User, Server) ->
-    Mod = gen_mod:db_mod(Server, ?MODULE),
+    Mod = gen_mod:ram_db_mod(Server, ?MODULE),
     Mod:list(User, Server).
 
 depends(_Host, _Opts) ->
     [].
 
 mod_opt_type(iqdisc) -> fun gen_iq_handler:check_type/1;
-mod_opt_type(db_type) -> fun(T) -> ejabberd_config:v_db(?MODULE, T) end;
-mod_opt_type(_) -> [db_type, iqdisc].
+mod_opt_type(ram_db_type) -> fun(T) -> ejabberd_config:v_db(?MODULE, T) end;
+mod_opt_type(_) -> [ram_db_type, iqdisc].

src/mod_carboncopy_redis.erl

@@ -0,0 +1,105 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% Created : 30 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2017   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(mod_carboncopy_redis).
+-behaviour(mod_carboncopy).
+
+%% API
+-export([init/2, enable/4, disable/3, list/2]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+init(_Host, _Opts) ->
+    clean_table().
+
+enable(LUser, LServer, LResource, NS) ->
+    USKey = us_key(LUser, LServer),
+    NodeKey = node_key(),
+    JID = jid:encode({LUser, LServer, LResource}),
+    case ejabberd_redis:multi(
+	   fun() ->
+		   ejabberd_redis:hset(USKey, LResource, NS),
+		   ejabberd_redis:sadd(NodeKey, [JID])
+	   end) of
+	{ok, _} ->
+	    ok;
+	{error, _} ->
+	    {error, db_failure}
+    end.
+
+disable(LUser, LServer, LResource) ->
+    USKey = us_key(LUser, LServer),
+    NodeKey = node_key(),
+    JID = jid:encode({LUser, LServer, LResource}),
+    case ejabberd_redis:multi(
+	   fun() ->
+		   ejabberd_redis:hdel(USKey, [LResource]),
+		   ejabberd_redis:srem(NodeKey, [JID])
+	   end) of
+	{ok, _} ->
+	    ok;
+	{error, _} ->
+	    {error, db_failure}
+    end.
+
+list(LUser, LServer) ->
+    USKey = us_key(LUser, LServer),
+    case ejabberd_redis:hgetall(USKey) of
+	{ok, Vals} ->
+	    Vals;
+	{error, _} ->
+	    []
+    end.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+clean_table() ->
+    ?INFO_MSG("Cleaning Redis 'carboncopy' table...", []),
+    NodeKey = node_key(),
+    case ejabberd_redis:smembers(NodeKey) of
+	{ok, JIDs} ->
+	    ejabberd_redis:multi(
+	      fun() ->
+		      lists:foreach(
+			fun(JID) ->
+				{U, S, R} = jid:split(jid:decode(JID)),
+				USKey = us_key(U, S),
+				ejabberd_redis:hdel(USKey, [R])
+			end, JIDs)
+	      end);
+	{error, _} ->
+	    ok
+    end,
+    ejabberd_redis:del([NodeKey]),
+    ok.
+
+us_key(LUser, LServer) ->
+    <<"ejabberd:carboncopy:users:", LUser/binary, $@, LServer/binary>>.
+
+node_key() ->
+    Node = erlang:atom_to_binary(node(), latin1),
+    <<"ejabberd:carboncopy:nodes:", Node/binary>>.

src/mod_carboncopy_sql.erl

@@ -0,0 +1,93 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% Created : 29 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2017   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(mod_carboncopy_sql).
+-behaviour(mod_carboncopy).
+
+-compile([{parse_transform, ejabberd_sql_pt}]).
+
+%% API
+-export([init/2, enable/4, disable/3, list/2]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+-include("ejabberd_sql_pt.hrl").
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+init(Host, _Opts) ->
+    clean_table(Host).
+
+enable(LUser, LServer, LResource, NS) ->
+    NodeS = erlang:atom_to_binary(node(), latin1),
+    case ?SQL_UPSERT(LServer, "carboncopy",
+		     ["!username=%(LUser)s",
+		      "!resource=%(LResource)s",
+		      "namespace=%(NS)s",
+		      "node=%(NodeS)s"]) of
+	ok ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to update 'carboncopy' table: ~p", [Err]),
+	    Err
+    end.
+
+disable(LUser, LServer, LResource) ->
+    case ejabberd_sql:sql_query(
+	   LServer,
+	   ?SQL("delete from carboncopy where username=%(LUser)s "
+		"and resource=%(LResource)s")) of
+	{updated, _} ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to delete from 'carboncopy' table: ~p", [Err]),
+	    Err
+    end.
+
+list(LUser, LServer) ->
+    case ejabberd_sql:sql_query(
+	   LServer,
+	   ?SQL("select @(resource)s, @(namespace)s from carboncopy "
+		"where username=%(LUser)s")) of
+	{selected, Rows} ->
+	    Rows;
+	Err ->
+	    ?ERROR_MSG("failed to select from 'carboncopy' table: ~p", [Err]),
+	    []
+    end.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+clean_table(LServer) ->
+    NodeS = erlang:atom_to_binary(node(), latin1),
+    ?INFO_MSG("Cleaning SQL 'carboncopy' table...", []),
+    case ejabberd_sql:sql_query(
+	   LServer,
+	   ?SQL("delete from carboncopy where node=%(NodeS)s")) of
+	{updated, _} ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to clean 'carboncopy' table: ~p", [Err]),
+	    Err
+    end.

src/mod_client_state.erl

@@ -47,17 +47,17 @@
 -define(CSI_QUEUE_MAX, 100).
 
 -type csi_type() :: presence | chatstate | {pep, binary()}.
--type csi_queue() :: {non_neg_integer(), non_neg_integer(), map()}.
+-type csi_queue() :: {non_neg_integer(), map()}.
 -type csi_timestamp() :: {non_neg_integer(), erlang:timestamp()}.
+-type csi_key() :: {ljid(), csi_type()}.
+-type csi_element() :: {csi_timestamp(), stanza()}.
 -type c2s_state() :: ejabberd_c2s:state().
 -type filter_acc() :: {stanza() | drop, c2s_state()}.
 
 %%--------------------------------------------------------------------
 %% gen_mod callbacks.
 %%--------------------------------------------------------------------
-
 -spec start(binary(), gen_mod:opts()) -> ok.
-
 start(Host, Opts) ->
     QueuePresence =
 	gen_mod:get_opt(queue_presence, Opts,
@@ -92,7 +92,6 @@ start(Host, Opts) ->
     end.
 
 -spec stop(binary()) -> ok.
-
 stop(Host) ->
     QueuePresence =
 	gen_mod:get_module_opt(Host, ?MODULE, queue_presence,
@@ -165,7 +164,6 @@ reload(Host, NewOpts, _OldOpts) ->
     end.
 
 -spec mod_opt_type(atom()) -> fun((term()) -> term()) | [atom()].
-
 mod_opt_type(queue_presence) ->
     fun(B) when is_boolean(B) -> B end;
 mod_opt_type(queue_chat_states) ->
@@ -175,7 +173,6 @@ mod_opt_type(queue_pep) ->
 mod_opt_type(_) -> [queue_presence, queue_chat_states, queue_pep].
 
 -spec depends(binary(), gen_mod:opts()) -> [{module(), hard | soft}].
-
 depends(_Host, _Opts) ->
     [].
 
@@ -320,25 +317,21 @@ enqueue_stanza(Type, Stanza, #{csi_state := inactive,
 	    C2SState1 = flush_queue(C2SState),
 	    enqueue_stanza(Type, Stanza, C2SState1);
 	false ->
-	    #jid{luser = U, lserver = S} = xmpp:get_from(Stanza),
-	    Q1 = queue_in({U, S}, Type, Stanza, Q),
+	    From = jid:tolower(xmpp:get_from(Stanza)),
+	    Q1 = queue_in({From, Type}, Stanza, Q),
 	    {stop, {drop, C2SState#{csi_queue => Q1}}}
     end;
 enqueue_stanza(_Type, Stanza, State) ->
     {Stanza, State}.
 
 -spec dequeue_sender(jid(), c2s_state()) -> c2s_state().
-dequeue_sender(#jid{luser = U, lserver = S},
+dequeue_sender(#jid{luser = U, lserver = S} = Sender,
 	       #{csi_queue := Q, jid := JID} = C2SState) ->
     ?DEBUG("Flushing packets of ~s@~s from CSI queue of ~s",
 	   [U, S, jid:encode(JID)]),
-    case queue_take({U, S}, Q) of
-	{Stanzas, Q1} ->
-	    C2SState1 = flush_stanzas(C2SState, Stanzas),
-	    C2SState1#{csi_queue => Q1};
-      error ->
-	    C2SState
-    end.
+    {Elems, Q1} = queue_take(Sender, Q),
+    C2SState1 = flush_stanzas(C2SState, Elems),
+    C2SState1#{csi_queue => Q1}.
 
 -spec flush_queue(c2s_state()) -> c2s_state().
 flush_queue(#{csi_queue := Q, jid := JID} = C2SState) ->
@@ -350,7 +343,7 @@ flush_queue(#{csi_queue := Q, jid := JID} = C2SState) ->
 		    [{csi_type(), csi_timestamp(), stanza()}]) -> c2s_state().
 flush_stanzas(#{lserver := LServer} = C2SState, Elems) ->
     lists:foldl(
-      fun({_Type, Time, Stanza}, AccState) ->
+      fun({Time, Stanza}, AccState) ->
 	      Stanza1 = add_delay_info(Stanza, LServer, Time),
 	      ejabberd_c2s:send(AccState, Stanza1)
       end, C2SState, Elems).
@@ -379,48 +372,29 @@ get_pep_node(#message{} = Msg) ->
 %%--------------------------------------------------------------------
 -spec queue_new() -> csi_queue().
 queue_new() ->
-    {0, 0, #{}}.
+    {0, #{}}.
 
--spec queue_in(term(), term(), term(), csi_queue()) -> csi_queue().
-queue_in(Key, Type, Val, {N, Seq, Q}) ->
+-spec queue_in(csi_key(), stanza(), csi_queue()) -> csi_queue().
+queue_in(Key, Stanza, {Seq, Q}) ->
     Seq1 = Seq + 1,
     Time = {Seq1, p1_time_compat:timestamp()},
-    case maps:get(Key, Q, error) of
-	error ->
-	    Q1 = maps:put(Key, [{Type, Time, Val}], Q),
-	    {N + 1, Seq1, Q1};
-	TypeVals ->
-	    case lists:keymember(Type, 1, TypeVals) of
-		true ->
-		    TypeVals1 = lists:keyreplace(
-				  Type, 1, TypeVals, {Type, Time, Val}),
-		    Q1 = maps:put(Key, TypeVals1, Q),
-		    {N, Seq1, Q1};
-		false ->
-		    TypeVals1 = [{Type, Time, Val}|TypeVals],
-		    Q1 = maps:put(Key, TypeVals1, Q),
-		    {N + 1, Seq1, Q1}
-	    end
-    end.
+    Q1 = maps:put(Key, {Time, Stanza}, Q),
+    {Seq1, Q1}.
 
--spec queue_take(term(), csi_queue()) -> {list(), csi_queue()} | error.
-queue_take(Key, {N, Seq, Q}) ->
-    case maps:get(Key, Q, error) of
-	error ->
-	    error;
-	TypeVals ->
-	    Q1 = maps:remove(Key, Q),
-	    {lists:keysort(2, TypeVals), {N-length(TypeVals), Seq, Q1}}
-    end.
+-spec queue_take(jid(), csi_queue()) -> {[csi_element()], csi_queue()}.
+queue_take(#jid{luser = LUser, lserver = LServer}, {Seq, Q}) ->
+    {Vals, Q1} = maps:fold(fun({{U, S, _}, _} = Key, Val, {AccVals, AccQ})
+				   when U == LUser, S == LServer ->
+				   {[Val | AccVals], maps:remove(Key, AccQ)};
+			       (_, _, Acc) ->
+				   Acc
+			    end, {[], Q}, Q),
+    {lists:keysort(1, Vals), {Seq, Q1}}.
 
 -spec queue_len(csi_queue()) -> non_neg_integer().
-queue_len({N, _, _}) ->
-    N.
+queue_len({_, Q}) ->
+    maps:size(Q).
 
--spec queue_to_list(csi_queue()) -> [term()].
-queue_to_list({_, _, Q}) ->
-    TypeVals = maps:fold(
-		 fun(_, Vals, Acc) ->
-			 Vals ++ Acc
-		 end, [], Q),
-    lists:keysort(2, TypeVals).
+-spec queue_to_list(csi_queue()) -> [csi_element()].
+queue_to_list({_, Q}) ->
+    lists:keysort(1, maps:values(Q)).

src/mod_configure.erl

@@ -919,7 +919,7 @@ get_form(Host,
 			  ENode/binary>>,
 		Instr = ?T(Lang, <<"Choose modules to stop">>),
 		Fs = lists:map(fun(M) ->
-				       S = jlib:atom_to_binary(M),
+				       S = misc:atom_to_binary(M),
 				       ?XFIELD(boolean, S, S, <<"0">>)
 			       end, SModules),
 		{result, #xdata{title = Title,
@@ -1224,7 +1224,7 @@ set_form(_From, _Host,
       Node ->
 	  lists:foreach(
 	    fun(#xdata_field{var = SVar, values = SVals}) ->
-		    Table = jlib:binary_to_atom(SVar),
+		    Table = misc:binary_to_atom(SVar),
 		    Type = case SVals of
 			       [<<"unknown">>] -> unknown;
 			       [<<"ram_copies">>] -> ram_copies;
@@ -1258,7 +1258,7 @@ set_form(_From, Host,
 	    fun(#xdata_field{var = Var, values = Vals}) ->
 		    case Vals of
 			[<<"1">>] ->
-			    Module = jlib:binary_to_atom(Var),
+			    Module = misc:binary_to_atom(Var),
 			    ejabberd_cluster:call(Node, gen_mod, stop_module,
 						  [Host, Module]);
 			_ -> ok
@@ -1657,7 +1657,7 @@ set_form(From, Host, ?NS_ADMINL(<<"user-stats">>), Lang,
 					       Server),
     IPs1 = [ejabberd_sm:get_user_ip(User, Server, Resource)
 	    || Resource <- Resources],
-    IPs = [<<(jlib:ip_to_list(IP))/binary, ":",
+    IPs = [<<(misc:ip_to_list(IP))/binary, ":",
              (integer_to_binary(Port))/binary>>
                || {IP, Port} <- IPs1],
     Items = ejabberd_hooks:run_fold(roster_get, Server, [],
@@ -1719,7 +1719,7 @@ stop_node(From, Host, ENode, Action, XData) ->
 	    mod_announce:announce_commands(empty, From, To, Request)
     end,
     Time = timer:seconds(Delay),
-    Node = jlib:binary_to_atom(ENode),
+    Node = misc:binary_to_atom(ENode),
     {ok, _} = timer:apply_after(Time, rpc, call, [Node, init, Action, []]),
     {result, undefined}.
 

src/mod_fail2ban.erl

@@ -166,7 +166,7 @@ code_change(_OldVsn, State, _Extra) ->
 -spec log_and_disconnect(ejabberd_c2s:state(), pos_integer(), non_neg_integer())
       -> {stop, ejabberd_c2s:state()}.
 log_and_disconnect(#{ip := {Addr, _}, lang := Lang} = State, Attempts, UnbanTS) ->
-    IP = jlib:ip_to_list(Addr),
+    IP = misc:ip_to_list(Addr),
     UnbanDate = format_date(
 		  calendar:now_to_universal_time(seconds_to_now(UnbanTS))),
     Format = <<"Too many (~p) failed authentications "

src/mod_http_api.erl

@@ -272,7 +272,7 @@ get_api_version([]) ->
 handle(Call, Auth, Args, Version) when is_atom(Call), is_list(Args) ->
     case ejabberd_commands:get_command_format(Call, Auth, Version) of
         {ArgsSpec, _} when is_list(ArgsSpec) ->
-            Args2 = [{jlib:binary_to_atom(Key), Value} || {Key, Value} <- Args],
+            Args2 = [{misc:binary_to_atom(Key), Value} || {Key, Value} <- Args],
             Spec = lists:foldr(
                     fun ({Key, binary}, Acc) ->
                             [{Key, <<>>}|Acc];
@@ -290,13 +290,13 @@ handle(Call, Auth, Args, Version) when is_atom(Call), is_list(Args) ->
 	    catch throw:not_found ->
 		    {404, <<"not_found">>};
 		  throw:{not_found, Why} when is_atom(Why) ->
-		    {404, jlib:atom_to_binary(Why)};
+		    {404, misc:atom_to_binary(Why)};
 		  throw:{not_found, Msg} ->
 		    {404, iolist_to_binary(Msg)};
 		  throw:not_allowed ->
 		    {401, <<"not_allowed">>};
 		  throw:{not_allowed, Why} when is_atom(Why) ->
-		    {401, jlib:atom_to_binary(Why)};
+		    {401, misc:atom_to_binary(Why)};
 		  throw:{not_allowed, Msg} ->
 		    {401, iolist_to_binary(Msg)};
                   throw:{error, account_unprivileged} ->
@@ -306,11 +306,11 @@ handle(Call, Auth, Args, Version) when is_atom(Call), is_list(Args) ->
 		  throw:{invalid_parameter, Msg} ->
 		    {400, iolist_to_binary(Msg)};
 		  throw:{error, Why} when is_atom(Why) ->
-		    {400, jlib:atom_to_binary(Why)};
+		    {400, misc:atom_to_binary(Why)};
 		  throw:{error, Msg} ->
 		    {400, iolist_to_binary(Msg)};
 		  throw:Error when is_atom(Error) ->
-		    {400, jlib:atom_to_binary(Error)};
+		    {400, misc:atom_to_binary(Error)};
 		  throw:Msg when is_list(Msg); is_binary(Msg) ->
 		    {400, iolist_to_binary(Msg)};
 		  _Error ->
@@ -456,45 +456,45 @@ format_command_result(Cmd, Auth, Result, Version) ->
     end.
 
 format_result(Atom, {Name, atom}) ->
-    {jlib:atom_to_binary(Name), jlib:atom_to_binary(Atom)};
+    {misc:atom_to_binary(Name), misc:atom_to_binary(Atom)};
 
 format_result(Int, {Name, integer}) ->
-    {jlib:atom_to_binary(Name), Int};
+    {misc:atom_to_binary(Name), Int};
 
 format_result([String | _] = StringList, {Name, string}) when is_list(String) ->
     Binarized = iolist_to_binary(string:join(StringList, "\n")),
-    {jlib:atom_to_binary(Name), Binarized};
+    {misc:atom_to_binary(Name), Binarized};
 
 format_result(String, {Name, string}) ->
-    {jlib:atom_to_binary(Name), iolist_to_binary(String)};
+    {misc:atom_to_binary(Name), iolist_to_binary(String)};
 
 format_result(Code, {Name, rescode}) ->
-    {jlib:atom_to_binary(Name), Code == true orelse Code == ok};
+    {misc:atom_to_binary(Name), Code == true orelse Code == ok};
 
 format_result({Code, Text}, {Name, restuple}) ->
-    {jlib:atom_to_binary(Name),
+    {misc:atom_to_binary(Name),
      {[{<<"res">>, Code == true orelse Code == ok},
        {<<"text">>, iolist_to_binary(Text)}]}};
 
 format_result(Code, {Name, restuple}) ->
-    {jlib:atom_to_binary(Name),
+    {misc:atom_to_binary(Name),
      {[{<<"res">>, Code == true orelse Code == ok},
        {<<"text">>, <<"">>}]}};
 
 format_result(Els, {Name, {list, {_, {tuple, [{_, atom}, _]}} = Fmt}}) ->
-    {jlib:atom_to_binary(Name), {[format_result(El, Fmt) || El <- Els]}};
+    {misc:atom_to_binary(Name), {[format_result(El, Fmt) || El <- Els]}};
 
 format_result(Els, {Name, {list, Def}}) ->
-    {jlib:atom_to_binary(Name), [element(2, format_result(El, Def)) || El <- Els]};
+    {misc:atom_to_binary(Name), [element(2, format_result(El, Def)) || El <- Els]};
 
 format_result(Tuple, {_Name, {tuple, [{_, atom}, ValFmt]}}) ->
     {Name2, Val} = Tuple,
     {_, Val2} = format_result(Val, ValFmt),
-    {jlib:atom_to_binary(Name2), Val2};
+    {misc:atom_to_binary(Name2), Val2};
 
 format_result(Tuple, {Name, {tuple, Def}}) ->
     Els = lists:zip(tuple_to_list(Tuple), Def),
-    {jlib:atom_to_binary(Name), {[format_result(El, ElDef) || {El, ElDef} <- Els]}};
+    {misc:atom_to_binary(Name), {[format_result(El, ElDef) || {El, ElDef} <- Els]}};
 
 format_result(404, {_Name, _}) ->
     "not_found".
@@ -537,7 +537,7 @@ json_error(HTTPCode, JSONCode, Message) ->
     }.
 
 log(Call, Args, {Addr, Port}) ->
-    AddrS = jlib:ip_to_list({Addr, Port}),
+    AddrS = misc:ip_to_list({Addr, Port}),
     ?INFO_MSG("API call ~s ~p from ~s:~p", [Call, Args, AddrS, Port]);
 log(Call, Args, IP) ->
     ?INFO_MSG("API call ~s ~p (~p)", [Call, Args, IP]).

src/mod_http_upload.erl

@@ -32,7 +32,7 @@
 -define(SLOT_TIMEOUT, 18000000). % 5 hours.
 -define(FORMAT(Error), file:format_error(Error)).
 -define(URL_ENC(URL), binary_to_list(ejabberd_http:url_encode(URL))).
--define(ADDR_TO_STR(IP), ejabberd_config:may_hide_data(jlib:ip_to_list(IP))).
+-define(ADDR_TO_STR(IP), ejabberd_config:may_hide_data(misc:ip_to_list(IP))).
 -define(STR_TO_INT(Str, B), binary_to_integer(iolist_to_binary(Str), B)).
 -define(DEFAULT_CONTENT_TYPE, <<"application/octet-stream">>).
 -define(CONTENT_TYPES,
@@ -500,16 +500,14 @@ get_proc_name(ServerHost, ModuleName) ->
 
 -spec expand_home(binary()) -> binary().
 
-expand_home(Subject) ->
+expand_home(Input) ->
     {ok, [[Home]]} = init:get_argument(home),
-    Parts = binary:split(Subject, <<"@HOME@">>, [global]),
-    str:join(Parts, list_to_binary(Home)).
+    misc:expand_keyword(<<"@HOME@">>, Input, Home).
 
 -spec expand_host(binary(), binary()) -> binary().
 
-expand_host(Subject, Host) ->
-    Parts = binary:split(Subject, <<"@HOST@">>, [global]),
-    str:join(Parts, Host).
+expand_host(Input, Host) ->
+    misc:expand_keyword(<<"@HOST@">>, Input, Host).
 
 %%--------------------------------------------------------------------
 %% Internal functions.
@@ -835,7 +833,6 @@ http_response(Host, Code, ExtraHeaders) ->
       -> {pos_integer(), [{binary(), binary()}], binary()}.
 
 http_response(Host, Code, ExtraHeaders, Body) ->
-    ServerHeader = {<<"Server">>, <<"ejabberd ", (?VERSION)/binary>>},
     CustomHeaders =
 	gen_mod:get_module_opt(Host, ?MODULE, custom_headers,
 			       fun(Headers) ->
@@ -847,10 +844,9 @@ http_response(Host, Code, ExtraHeaders, Body) ->
 			       []),
     Headers = case proplists:is_defined(<<"Content-Type">>, ExtraHeaders) of
 		  true ->
-		      [ServerHeader | ExtraHeaders];
+		      ExtraHeaders;
 		  false ->
-		      [ServerHeader, {<<"Content-Type">>, <<"text/plain">>} |
-		       ExtraHeaders]
+		      [{<<"Content-Type">>, <<"text/plain">>} | ExtraHeaders]
 	      end ++ CustomHeaders,
     {Code, Headers, Body}.
 

src/mod_irc_connection.erl

@@ -50,7 +50,6 @@
          encoding = <<"">>     :: binary(),
          port = 0              :: inet:port_number(),
          password = <<"">>     :: binary(),
-         queue = queue:new()   :: ?TQUEUE,
          user = #jid{}         :: jid(),
          host = <<"">>         :: binary(),
 	 server = <<"">>       :: binary(),
@@ -112,7 +111,7 @@ init([From, Host, Server, Username, Encoding, Port,
       Password, Ident, RemoteAddr, RealName, WebircPassword, Mod]) ->
     gen_fsm:send_event(self(), init),
     {ok, open_socket,
-     #state{queue = queue:new(), mod = Mod,
+     #state{mod = Mod,
 	    encoding = Encoding, port = Port, password = Password,
 	    user = From, nick = Username, host = Host,
 	    server = Server, ident = Ident, realname = RealName,
@@ -695,15 +694,6 @@ send_text(#state{socket = Socket, encoding = Encoding},
     CText = iconv:convert(<<"utf-8">>, Encoding, iolist_to_binary(Text)),
     gen_tcp:send(Socket, CText).
 
-%send_queue(Socket, Q) ->
-%    case queue:out(Q) of
-%	{{value, El}, Q1} ->
-%	    send_element(Socket, El),
-%	    send_queue(Socket, Q1);
-%	{empty, Q1} ->
-%	    ok
-%    end.
-
 bounce_messages(Reason) ->
     receive
 	{send_element, El} ->

src/mod_irc_sql.erl

@@ -55,7 +55,7 @@ get_data(LServer, Host, From) ->
 
 set_data(LServer, Host, From, Data) ->
     SJID = jid:encode(jid:tolower(jid:remove_resource(From))),
-    SData = jlib:term_to_expr(Data),
+    SData = misc:term_to_expr(Data),
     F = fun () ->
                 ?SQL_UPSERT_T(
                    "irc_custom",
@@ -73,7 +73,7 @@ export(_Server) ->
               case str:suffix(Host, IRCHost) of
                   true ->
                       SJID = jid:encode(jid:make(U, S)),
-                      SData = jlib:term_to_expr(Data),
+                      SData = misc:term_to_expr(Data),
                       [?SQL("delete from irc_custom"
                             " where jid=%(SJID)s and host=%(IRCHost)s;"),
                        ?SQL("insert into irc_custom(jid, host, data)"

src/mod_mam.erl

@@ -436,7 +436,7 @@ delete_old_messages(TypeBin, Days) when TypeBin == <<"chat">>;
 					TypeBin == <<"all">> ->
     Diff = Days * 24 * 60 * 60 * 1000000,
     TimeStamp = usec_to_now(p1_time_compat:system_time(micro_seconds) - Diff),
-    Type = jlib:binary_to_atom(TypeBin),
+    Type = misc:binary_to_atom(TypeBin),
     DBTypes = lists:usort(
 		lists:map(
 		  fun(Host) ->
@@ -518,7 +518,8 @@ process_iq(LServer, #iq{from = #jid{luser = LUser}, lang = Lang,
     end,
     case SubEl of
 	#mam_query{rsm = #rsm_set{index = I}} when is_integer(I) ->
-	    xmpp:make_error(IQ, xmpp:err_feature_not_implemented());
+	    Txt = <<"Unsupported <index/> element">>,
+	    xmpp:make_error(IQ, xmpp:err_feature_not_implemented(Txt, Lang));
 	#mam_query{rsm = RSM, xmlns = NS} ->
 	    case parse_query(SubEl, Lang) of
 		{ok, Query} ->
@@ -837,7 +838,8 @@ select(_LServer, JidRequestor, JidArchive, Query, RSM,
 				 history = History}} = MsgType) ->
     Start = proplists:get_value(start, Query),
     End = proplists:get_value('end', Query),
-    #lqueue{len = L, queue = Q} = History,
+    #lqueue{queue = Q} = History,
+    L = p1_queue:len(Q),
     Msgs =
 	lists:flatmap(
 	  fun({Nick, Pkt, _HaveSubject, Now, _Size}) ->
@@ -861,7 +863,7 @@ select(_LServer, JidRequestor, JidArchive, Query, RSM,
 		      false ->
 			  []
 		  end
-	  end, queue:to_list(Q)),
+	  end, p1_queue:to_list(Q)),
     case RSM of
 	#rsm_set{max = Max, before = Before} when is_binary(Before) ->
 	    {NewMsgs, IsComplete} = filter_by_max(lists:reverse(Msgs), Max),

src/mod_mam_sql.erl

@@ -58,7 +58,7 @@ remove_room(LServer, LName, LHost) ->
 
 delete_old_messages(ServerHost, TimeStamp, Type) ->
     TypeClause = if Type == all -> <<"">>;
-		    true -> [<<" and kind='">>, jlib:atom_to_binary(Type), <<"'">>]
+		    true -> [<<" and kind='">>, misc:atom_to_binary(Type), <<"'">>]
 		 end,
     TS = integer_to_binary(now_to_usec(TimeStamp)),
     ejabberd_sql:sql_query(
@@ -83,7 +83,7 @@ store(Pkt, LServer, {LUser, LHost}, Type, Peer, Nick, _Dir) ->
 	      jid:tolower(Peer)),
     XML = fxml:element_to_binary(Pkt),
     Body = fxml:get_subtag_cdata(Pkt, <<"body">>),
-    SType = jlib:atom_to_binary(Type),
+    SType = misc:atom_to_binary(Type),
     case ejabberd_sql:sql_query(
            LServer,
            ?SQL("insert into archive (username, timestamp,"
@@ -107,8 +107,8 @@ write_prefs(LUser, _LServer, #archive_prefs{default = Default,
 					   always = Always},
 	    ServerHost) ->
     SDefault = erlang:atom_to_binary(Default, utf8),
-    SAlways = jlib:term_to_expr(Always),
-    SNever = jlib:term_to_expr(Never),
+    SAlways = misc:term_to_expr(Always),
+    SNever = misc:term_to_expr(Never),
     case ?SQL_UPSERT(
             ServerHost,
             "archive_prefs",
@@ -321,7 +321,7 @@ make_archive_el(TS, XML, Peer, Kind, Nick, MsgType, JidRequestor, JidArchive) ->
 			    T = case Kind of
 				    <<"">> -> chat;
 				    null -> chat;
-				    _ -> jlib:binary_to_atom(Kind)
+				    _ -> misc:binary_to_atom(Kind)
 				end,
 			    mod_mam:msg_to_el(
 			      #archive_msg{timestamp = Now,

src/mod_metrics.erl

@@ -133,7 +133,7 @@ send_metrics(Host, Probe, Peer, Port) ->
     % grapherl metrics are named first with service domain, then nodename
     % and name of the data itself, followed by type timestamp and value
     % example => process-one.net/xmpp-1.user_receive_packet:c/1441784958:1
-    [_, NodeId] = str:tokens(jlib:atom_to_binary(node()), <<"@">>),
+    [_, NodeId] = str:tokens(misc:atom_to_binary(node()), <<"@">>),
     [Node | _] = str:tokens(NodeId, <<".">>),
     BaseId = <<Host/binary, "/", Node/binary, ".">>,
     DateTime = erlang:universaltime(),
@@ -144,11 +144,11 @@ send_metrics(Host, Probe, Peer, Port) ->
 	    case Probe of
 		{Key, Val} ->
 		    BVal = integer_to_binary(Val),
-		    Data = <<BaseId/binary, (jlib:atom_to_binary(Key))/binary,
+		    Data = <<BaseId/binary, (misc:atom_to_binary(Key))/binary,
 			    ":g/", TS/binary, ":", BVal/binary>>,
 		    gen_udp:send(Socket, Peer, Port, Data);
 		Key ->
-		    Data = <<BaseId/binary, (jlib:atom_to_binary(Key))/binary,
+		    Data = <<BaseId/binary, (misc:atom_to_binary(Key))/binary,
 			    ":c/", TS/binary, ":1">>,
 		    gen_udp:send(Socket, Peer, Port, Data)
 	    end,

src/mod_muc.erl

@@ -80,6 +80,7 @@
          access = {none, none, none, none} :: {atom(), atom(), atom(), atom()},
          history_size = 20 :: non_neg_integer(),
          max_rooms_discoitems = 100 :: non_neg_integer(),
+	 queue_type = ram :: ram | file,
          default_room_opts = [] :: list(),
          room_shaper = none :: shaper:shaper()}).
 
@@ -93,16 +94,16 @@
 -callback get_rooms(binary(), binary()) -> [#muc_room{}].
 -callback get_nick(binary(), binary(), jid()) -> binary() | error.
 -callback set_nick(binary(), binary(), jid(), binary()) -> {atomic, ok | false}.
--callback register_online_room(binary(), binary(), pid()) -> any().
--callback unregister_online_room(binary(), binary(), pid()) -> any().
--callback find_online_room(binary(), binary()) -> {ok, pid()} | error.
--callback get_online_rooms(binary(), undefined | rsm_set()) -> [{binary(), binary(), pid()}].
--callback count_online_rooms(binary()) -> non_neg_integer().
+-callback register_online_room(binary(), binary(), binary(), pid()) -> any().
+-callback unregister_online_room(binary(), binary(), binary(), pid()) -> any().
+-callback find_online_room(binary(), binary(), binary()) -> {ok, pid()} | error.
+-callback get_online_rooms(binary(), binary(), undefined | rsm_set()) -> [{binary(), binary(), pid()}].
+-callback count_online_rooms(binary(), binary()) -> non_neg_integer().
 -callback rsm_supported() -> boolean().
--callback register_online_user(ljid(), binary(), binary()) -> any().
--callback unregister_online_user(ljid(), binary(), binary()) -> any().
--callback count_online_rooms_by_user(binary(), binary()) -> non_neg_integer().
--callback get_online_rooms_by_user(binary(), binary()) -> [{binary(), binary()}].
+-callback register_online_user(binary(), ljid(), binary(), binary()) -> any().
+-callback unregister_online_user(binary(), ljid(), binary(), binary()) -> any().
+-callback count_online_rooms_by_user(binary(), binary(), binary()) -> non_neg_integer().
+-callback get_online_rooms_by_user(binary(), binary(), binary()) -> [{binary(), binary()}].
 
 %%====================================================================
 %% API
@@ -126,7 +127,7 @@ shutdown_rooms(Host) ->
     RMod = gen_mod:ram_db_mod(Host, ?MODULE),
     MyHost = gen_mod:get_module_opt_host(Host, mod_muc,
 					 <<"conference.@HOST@">>),
-    Rooms = RMod:get_online_rooms(MyHost, undefined),
+    Rooms = RMod:get_online_rooms(Host, MyHost, undefined),
     lists:flatmap(
       fun({_, _, Pid}) when node(Pid) == node() ->
 	      Pid ! shutdown,
@@ -179,13 +180,13 @@ can_use_nick(ServerHost, Host, JID, Nick) ->
 find_online_room(Room, Host) ->
     ServerHost = ejabberd_router:host_of_route(Host),
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:find_online_room(Room, Host).
+    RMod:find_online_room(ServerHost, Room, Host).
 
 -spec register_online_room(binary(), binary(), pid()) -> any().
 register_online_room(Room, Host, Pid) ->
     ServerHost = ejabberd_router:host_of_route(Host),
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:register_online_room(Room, Host, Pid).
+    RMod:register_online_room(ServerHost, Room, Host, Pid).
 
 -spec get_online_rooms(binary()) -> [{binary(), binary(), pid()}].
 get_online_rooms(Host) ->
@@ -200,22 +201,22 @@ count_online_rooms(Host) ->
 -spec register_online_user(binary(), ljid(), binary(), binary()) -> any().
 register_online_user(ServerHost, LJID, Name, Host) ->
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:register_online_user(LJID, Name, Host).
+    RMod:register_online_user(ServerHost, LJID, Name, Host).
 
 -spec unregister_online_user(binary(), ljid(), binary(), binary()) -> any().
 unregister_online_user(ServerHost, LJID, Name, Host) ->
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:unregister_online_user(LJID, Name, Host).
+    RMod:unregister_online_user(ServerHost, LJID, Name, Host).
 
 -spec count_online_rooms_by_user(binary(), binary(), binary()) -> non_neg_integer().
 count_online_rooms_by_user(ServerHost, LUser, LServer) ->
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:count_online_rooms_by_user(LUser, LServer).
+    RMod:count_online_rooms_by_user(ServerHost, LUser, LServer).
 
 -spec get_online_rooms_by_user(binary(), binary(), binary()) -> [{binary(), binary()}].
 get_online_rooms_by_user(ServerHost, LUser, LServer) ->
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:get_online_rooms_by_user(LUser, LServer).
+    RMod:get_online_rooms_by_user(ServerHost, LUser, LServer).
 
 %%====================================================================
 %% gen_server callbacks
@@ -226,7 +227,7 @@ init([Host, Opts]) ->
     IQDisc = gen_mod:get_opt(iqdisc, Opts, fun gen_iq_handler:check_type/1,
                              one_queue),
     #state{access = Access, host = MyHost,
-	   history_size = HistorySize,
+	   history_size = HistorySize, queue_type = QueueType,
 	   room_shaper = RoomShaper} = State = init_state(Host, Opts),
     Mod = gen_mod:db_mod(Host, Opts, ?MODULE),
     RMod = gen_mod:ram_db_mod(Host, Opts, ?MODULE),
@@ -234,7 +235,7 @@ init([Host, Opts]) ->
     RMod:init(Host, [{host, MyHost}|Opts]),
     register_iq_handlers(MyHost, IQDisc),
     ejabberd_router:register_route(MyHost, Host),
-    load_permanent_rooms(MyHost, Host, Access, HistorySize, RoomShaper),
+    load_permanent_rooms(MyHost, Host, Access, HistorySize, RoomShaper, QueueType),
     {ok, State}.
 
 handle_call(stop, _From, State) ->
@@ -242,7 +243,7 @@ handle_call(stop, _From, State) ->
 handle_call({create, Room, From, Nick, Opts}, _From,
 	    #state{host = Host, server_host = ServerHost,
 		   access = Access, default_room_opts = DefOpts,
-		   history_size = HistorySize,
+		   history_size = HistorySize, queue_type = QueueType,
 		   room_shaper = RoomShaper} = State) ->
     ?DEBUG("MUC: create new room '~s'~n", [Room]),
     NewOpts = case Opts of
@@ -253,9 +254,9 @@ handle_call({create, Room, From, Nick, Opts}, _From,
 		  Host, ServerHost, Access,
 		  Room, HistorySize,
 		  RoomShaper, From,
-		  Nick, NewOpts),
+		  Nick, NewOpts, QueueType),
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:register_online_room(Room, Host, Pid),
+    RMod:register_online_room(ServerHost, Room, Host, Pid),
     {reply, ok, State}.
 
 handle_cast({reload, ServerHost, NewOpts, OldOpts}, #state{host = OldHost}) ->
@@ -300,13 +301,14 @@ handle_cast(Msg, State) ->
 handle_info({route, Packet},
 	    #state{host = Host, server_host = ServerHost,
 		   access = Access, default_room_opts = DefRoomOpts,
-		   history_size = HistorySize,
+		   history_size = HistorySize, queue_type = QueueType,
 		   max_rooms_discoitems = MaxRoomsDiscoItems,
 		   room_shaper = RoomShaper} = State) ->
     From = xmpp:get_from(Packet),
     To = xmpp:get_to(Packet),
     case catch do_route(Host, ServerHost, Access, HistorySize, RoomShaper,
-			From, To, Packet, DefRoomOpts, MaxRoomsDiscoItems) of
+			From, To, Packet, DefRoomOpts, MaxRoomsDiscoItems,
+			QueueType) of
 	{'EXIT', Reason} ->
 	    ?ERROR_MSG("~p", [Reason]);
 	_ ->
@@ -316,7 +318,7 @@ handle_info({route, Packet},
 handle_info({room_destroyed, {Room, Host}, Pid}, State) ->
     ServerHost = State#state.server_host,
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:unregister_online_room(Room, Host, Pid),
+    RMod:unregister_online_room(ServerHost, Room, Host, Pid),
     {noreply, State};
 handle_info(Info, State) ->
     ?ERROR_MSG("unexpected info: ~p", [Info]),
@@ -353,6 +355,13 @@ init_state(Host, Opts) ->
     DefRoomOpts1 = gen_mod:get_opt(default_room_options, Opts,
 				   fun(L) when is_list(L) -> L end,
 				   []),
+    QueueType = case gen_mod:get_opt(queue_type, Opts,
+				     mod_opt_type(queue_type)) of
+		    undefined ->
+			ejabberd_config:default_queue_type(Host);
+		    Type ->
+			Type
+		end,
     DefRoomOpts =
 	lists:flatmap(
 	  fun({Opt, Val}) ->
@@ -410,6 +419,7 @@ init_state(Host, Opts) ->
 	   server_host = Host,
 	   access = {Access, AccessCreate, AccessAdmin, AccessPersistent},
 	   default_room_opts = DefRoomOpts,
+	   queue_type = QueueType,
 	   history_size = HistorySize,
 	   max_rooms_discoitems = MaxRoomsDiscoItems,
 	   room_shaper = RoomShaper}.
@@ -437,12 +447,12 @@ unregister_iq_handlers(Host) ->
     gen_iq_handler:remove_iq_handler(ejabberd_local, Host, ?NS_DISCO_ITEMS).
 
 do_route(Host, ServerHost, Access, HistorySize, RoomShaper,
-	 From, To, Packet, DefRoomOpts, _MaxRoomsDiscoItems) ->
+	 From, To, Packet, DefRoomOpts, _MaxRoomsDiscoItems, QueueType) ->
     {AccessRoute, _AccessCreate, _AccessAdmin, _AccessPersistent} = Access,
     case acl:match_rule(ServerHost, AccessRoute, From) of
 	allow ->
 	    do_route1(Host, ServerHost, Access, HistorySize, RoomShaper,
-		      From, To, Packet, DefRoomOpts);
+		      From, To, Packet, DefRoomOpts, QueueType);
 	deny ->
 	    Lang = xmpp:get_lang(Packet),
 	    ErrText = <<"Access denied by service policy">>,
@@ -452,11 +462,11 @@ do_route(Host, ServerHost, Access, HistorySize, RoomShaper,
 
 do_route1(_Host, _ServerHost, _Access, _HistorySize, _RoomShaper,
 	  _From, #jid{luser = <<"">>, lresource = <<"">>} = _To,
-	  #iq{} = IQ, _DefRoomOpts) ->
+	  #iq{} = IQ, _DefRoomOpts, _QueueType) ->
     ejabberd_local:process_iq(IQ);
 do_route1(Host, ServerHost, Access, _HistorySize, _RoomShaper,
 	  From, #jid{luser = <<"">>, lresource = <<"">>} = _To,
-	  #message{lang = Lang, body = Body, type = Type} = Packet, _) ->
+	  #message{lang = Lang, body = Body, type = Type} = Packet, _, _) ->
     {_AccessRoute, _AccessCreate, AccessAdmin, _AccessPersistent} = Access,
     if Type == error ->
 	    ok;
@@ -473,15 +483,15 @@ do_route1(Host, ServerHost, Access, _HistorySize, _RoomShaper,
 	    end
     end;
 do_route1(_Host, _ServerHost, _Access, _HistorySize, _RoomShaper,
-	  _From, #jid{luser = <<"">>} = _To, Packet, _DefRoomOpts) ->
+	  _From, #jid{luser = <<"">>} = _To, Packet, _DefRoomOpts, _) ->
     Err = xmpp:err_service_unavailable(),
     ejabberd_router:route_error(Packet, Err);
 do_route1(Host, ServerHost, Access, HistorySize, RoomShaper,
-	  From, To, Packet, DefRoomOpts) ->
+	  From, To, Packet, DefRoomOpts, QueueType) ->
     {_AccessRoute, AccessCreate, _AccessAdmin, _AccessPersistent} = Access,
     {Room, _, Nick} = jid:tolower(To),
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    case RMod:find_online_room(Room, Host) of
+    case RMod:find_online_room(ServerHost, Room, Host) of
 	error ->
 	    case is_create_request(Packet) of
 		true ->
@@ -492,8 +502,9 @@ do_route1(Host, ServerHost, Access, HistorySize, RoomShaper,
 			    {ok, Pid} = start_new_room(
 					  Host, ServerHost, Access,
 					  Room, HistorySize,
-					  RoomShaper, From, Nick, DefRoomOpts),
-			    RMod:register_online_room(Room, Host, Pid),
+					  RoomShaper, From, Nick, DefRoomOpts,
+					  QueueType),
+			    RMod:register_online_room(ServerHost, Room, Host, Pid),
 			    mod_muc_room:route(Pid, Packet),
 			    ok;
 			false ->
@@ -659,18 +670,18 @@ get_rooms(ServerHost, Host) ->
     Mod:get_rooms(LServer, Host).
 
 load_permanent_rooms(Host, ServerHost, Access,
-		     HistorySize, RoomShaper) ->
+		     HistorySize, RoomShaper, QueueType) ->
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
     lists:foreach(
       fun(R) ->
 		{Room, Host} = R#muc_room.name_host,
-	      case RMod:find_online_room(Room, Host) of
+	      case RMod:find_online_room(ServerHost, Room, Host) of
 		  error ->
 			{ok, Pid} = mod_muc_room:start(Host,
 				ServerHost, Access, Room,
 				HistorySize, RoomShaper,
-				R#muc_room.opts),
-		      RMod:register_online_room(Room, Host, Pid);
+				R#muc_room.opts, QueueType),
+		      RMod:register_online_room(ServerHost, Room, Host, Pid);
 		  {ok, _} ->
 		      ok
 		end
@@ -679,17 +690,17 @@ load_permanent_rooms(Host, ServerHost, Access,
 
 start_new_room(Host, ServerHost, Access, Room,
 	    HistorySize, RoomShaper, From,
-	    Nick, DefRoomOpts) ->
+	    Nick, DefRoomOpts, QueueType) ->
     case restore_room(ServerHost, Host, Room) of
 	error ->
 	    ?DEBUG("MUC: open new room '~s'~n", [Room]),
 	    mod_muc_room:start(Host, ServerHost, Access, Room,
 		HistorySize, RoomShaper,
-		From, Nick, DefRoomOpts);
+		From, Nick, DefRoomOpts, QueueType);
 	Opts ->
 	    ?DEBUG("MUC: restore room '~s'~n", [Room]),
 	    mod_muc_room:start(Host, ServerHost, Access, Room,
-		HistorySize, RoomShaper, Opts)
+		HistorySize, RoomShaper, Opts, QueueType)
     end.
 
 -spec iq_disco_items(binary(), binary(), jid(), binary(), integer(), binary(),
@@ -845,12 +856,12 @@ get_online_rooms(ServerHost, Host) ->
 			  [{binary(), binary(), pid()}].
 get_online_rooms(ServerHost, Host, RSM) ->
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:get_online_rooms(Host, RSM).
+    RMod:get_online_rooms(ServerHost, Host, RSM).
 
 -spec count_online_rooms(binary(), binary()) -> non_neg_integer().
 count_online_rooms(ServerHost, Host) ->
     RMod = gen_mod:ram_db_mod(ServerHost, ?MODULE),
-    RMod:count_online_rooms(Host).
+    RMod:count_online_rooms(ServerHost, Host).
 
 opts_to_binary(Opts) ->
     lists:map(
@@ -954,11 +965,13 @@ mod_opt_type(user_message_shaper) ->
     fun (A) when is_atom(A) -> A end;
 mod_opt_type(user_presence_shaper) ->
     fun (A) when is_atom(A) -> A end;
+mod_opt_type(queue_type) ->
+    fun(ram) -> ram; (file) -> file end;
 mod_opt_type(_) ->
     [access, access_admin, access_create, access_persistent,
      db_type, ram_db_type, default_room_options, history_size, host,
      max_room_desc, max_room_id, max_room_name,
      max_rooms_discoitems, max_user_conferences, max_users,
      max_users_admin_threshold, max_users_presence,
-     min_message_interval, min_presence_interval,
+     min_message_interval, min_presence_interval, queue_type,
      regexp_room_id, room_shaper, user_message_shaper, user_presence_shaper].

src/mod_muc_admin.erl

@@ -29,8 +29,8 @@
 -behaviour(gen_mod).
 
 -export([start/2, stop/1, reload/3, depends/2, muc_online_rooms/1,
-	 muc_unregister_nick/1, create_room/3, destroy_room/2,
-	 create_room_with_opts/4,
+	 muc_register_nick/3, muc_unregister_nick/1,
+	 create_room_with_opts/4, create_room/3, destroy_room/2,
 	 create_rooms_file/1, destroy_rooms_file/1,
 	 rooms_unused_list/2, rooms_unused_destroy/2,
 	 get_user_rooms/2, get_room_occupants/2,
@@ -44,6 +44,7 @@
 -include("ejabberd.hrl").
 -include("logger.hrl").
 -include("xmpp.hrl").
+-include("mod_muc.hrl").
 -include("mod_muc_room.hrl").
 -include("ejabberd_http.hrl").
 -include("ejabberd_web_admin.hrl").
@@ -85,6 +86,13 @@ get_commands_spec() ->
 		       module = ?MODULE, function = muc_online_rooms,
 		       args = [{host, binary}],
 		       result = {rooms, {list, {room, string}}}},
+     #ejabberd_commands{name = muc_register_nick, tags = [muc],
+		       desc = "Register a nick in the MUC service",
+		       longdesc = "Provide the nick, the user JID and the MUC service",
+		       module = ?MODULE, function = muc_register_nick,
+		       args = [{nick, binary}, {jid, binary}, {domain, binary}],
+		       args_example = [<<"Tim">>, <<"tim@example.org">>, <<"conference.example.org">>],
+		       result = {res, rescode}},
      #ejabberd_commands{name = muc_unregister_nick, tags = [muc],
 		       desc = "Unregister the nick in the MUC service",
 		       module = ?MODULE, function = muc_unregister_nick,
@@ -233,9 +241,20 @@ muc_online_rooms(ServerHost) ->
 	       || {Name, _, _} <- mod_muc:get_online_rooms(Host)]
       end, Hosts).
 
+muc_register_nick(Nick, JIDBinary, Domain) ->
+    JID = jlib:string_to_jid(JIDBinary),
+    %{jid, UID, Host, _,_,_,_} = jlib:string_to_jid(JIDBinary),
+    F = fun (MHost, MNick) ->
+	    mnesia:write(#muc_registered{us_host=MHost, nick=MNick})
+	end,
+    case mnesia:transaction(F, [{{JID#jid.luser, JID#jid.lserver}, Domain}, Nick]) of
+	{atomic, ok} -> ok;
+	{aborted, _Error} -> error
+    end.
+
 muc_unregister_nick(Nick) ->
     F2 = fun(N) ->
-		 [{_,Key,_}] = mnesia:index_read(muc_registered, N, 3),
+		 [{_,Key,_}|_] = mnesia:index_read(muc_registered, N, 3),
 		 mnesia:delete({muc_registered, Key})
 	 end,
     case mnesia:transaction(F2, [Nick], 1) of
@@ -431,10 +450,10 @@ prepare_room_info(Room_info) ->
     [NameHost,
      integer_to_binary(Num_participants),
      Ts_last_message,
-     jlib:atom_to_binary(Public),
-     jlib:atom_to_binary(Persistent),
-     jlib:atom_to_binary(Logging),
-     jlib:atom_to_binary(Just_created),
+     misc:atom_to_binary(Public),
+     misc:atom_to_binary(Persistent),
+     misc:atom_to_binary(Logging),
+     misc:atom_to_binary(Just_created),
      Title].
 
 
@@ -471,6 +490,8 @@ create_room_with_opts(Name1, Host1, ServerHost, CustomRoomOpts) ->
     AcPer = gen_mod:get_module_opt(ServerHost, mod_muc, access_persistent, fun(X) -> X end, all),
     HistorySize = gen_mod:get_module_opt(ServerHost, mod_muc, history_size, fun(X) -> X end, 20),
     RoomShaper = gen_mod:get_module_opt(ServerHost, mod_muc, room_shaper, fun(X) -> X end, none),
+    QueueType = gen_mod:get_module_opt(ServerHost, mod_muc, queue_type, fun(X) -> X end,
+				       ejabberd_config:default_queue_type(ServerHost)),
 
     %% If the room does not exist yet in the muc_online_room
     case mod_muc:find_online_room(Name, Host) of
@@ -483,7 +504,8 @@ create_room_with_opts(Name1, Host1, ServerHost, CustomRoomOpts) ->
 			  Name,
 			  HistorySize,
 			  RoomShaper,
-			  RoomOpts),
+			  RoomOpts,
+			  QueueType),
 	    mod_muc:register_online_room(Name, Host, Pid),
 	    ok;
 	{ok, _} ->
@@ -801,7 +823,7 @@ change_room_option(Name, Service, OptionString, ValueString) ->
     end.
 
 format_room_option(OptionString, ValueString) ->
-    Option = jlib:binary_to_atom(OptionString),
+    Option = misc:binary_to_atom(OptionString),
     Value = case Option of
 		title -> ValueString;
 		description -> ValueString;
@@ -809,7 +831,7 @@ format_room_option(OptionString, ValueString) ->
 		subject ->ValueString;
 		subject_author ->ValueString;
 		max_users -> binary_to_integer(ValueString);
-		_ -> jlib:binary_to_atom(ValueString)
+		_ -> misc:binary_to_atom(ValueString)
 	    end,
     {Option, Value}.
 
@@ -870,9 +892,9 @@ get_room_options(Pid) ->
     get_options(Config).
 
 get_options(Config) ->
-    Fields = [jlib:atom_to_binary(Field) || Field <- record_info(fields, config)],
+    Fields = [misc:atom_to_binary(Field) || Field <- record_info(fields, config)],
     [config | ValuesRaw] = tuple_to_list(Config),
-    Values = lists:map(fun(V) when is_atom(V) -> jlib:atom_to_binary(V);
+    Values = lists:map(fun(V) when is_atom(V) -> misc:atom_to_binary(V);
                           (V) when is_integer(V) -> integer_to_binary(V);
                           (V) when is_tuple(V); is_list(V) -> list_to_binary(hd(io_lib:format("~w", [V])));
                           (V) -> V end, ValuesRaw),
@@ -914,7 +936,7 @@ get_room_affiliations(Name, Service) ->
 %% If the affiliation is 'none', the action is to remove,
 %% In any other case the action will be to create the affiliation.
 set_room_affiliation(Name, Service, JID, AffiliationString) ->
-    Affiliation = jlib:binary_to_atom(AffiliationString),
+    Affiliation = misc:binary_to_atom(AffiliationString),
     case mod_muc:find_online_room(Name, Service) of
 	{ok, Pid} ->
 	    %% Get the PID for the online room so we can get the state of the room

src/mod_muc_log.erl

@@ -537,7 +537,7 @@ make_dir_rec(Dir) ->
 %% {ok, F1}=file:open("valid-xhtml10.png", [read]).
 %% {ok, F1b}=file:read(F1, 1000000).
 %% c("../../ejabberd/src/jlib.erl").
-%% jlib:encode_base64(F1b).
+%% misc:encode_base64(F1b).
 
 image_base64(<<"powered-by-erlang.png">>) ->
     <<"iVBORw0KGgoAAAANSUhEUgAAAGUAAAAfCAYAAAD+xQNoA"
@@ -713,7 +713,7 @@ create_image_files(Images_dir) ->
     lists:foreach(fun (Filename) ->
 			  Filename_full = fjoin([Images_dir, Filename]),
 			  {ok, F} = file:open(Filename_full, [write]),
-			  Image = jlib:decode_base64(image_base64(Filename)),
+			  Image = misc:decode_base64(image_base64(Filename)),
 			  io:format(F, <<"~s">>, [Image]),
 			  file:close(F)
 		  end,
@@ -1051,7 +1051,7 @@ roomconfig_to_string(Options, Lang, FileFormat) ->
 					   allow_private_messages_from_visitors ->
 					       <<"<div class=\"rcot\">",
 						 OptText/binary, ": \"",
-						 (htmlize(?T(jlib:atom_to_binary(T)),
+						 (htmlize(?T(misc:atom_to_binary(T)),
 							  FileFormat))/binary,
 						 "\"</div>">>;
 					   _ -> <<"\"", T/binary, "\"">>

src/mod_muc_mnesia.erl

@@ -30,10 +30,10 @@
 %% API
 -export([init/2, import/3, store_room/4, restore_room/3, forget_room/3,
 	 can_use_nick/4, get_rooms/2, get_nick/3, set_nick/4]).
--export([register_online_room/3, unregister_online_room/3, find_online_room/2,
-	 get_online_rooms/2, count_online_rooms/1, rsm_supported/0,
-	 register_online_user/3, unregister_online_user/3,
-	 count_online_rooms_by_user/2, get_online_rooms_by_user/2]).
+-export([register_online_room/4, unregister_online_room/4, find_online_room/3,
+	 get_online_rooms/3, count_online_rooms/2, rsm_supported/0,
+	 register_online_user/4, unregister_online_user/4,
+	 count_online_rooms_by_user/3, get_online_rooms_by_user/3]).
 -export([set_affiliation/6, set_affiliations/4, get_affiliation/5,
 	 get_affiliations/3, search_affiliation/4]).
 %% gen_server callbacks
@@ -157,27 +157,30 @@ get_affiliations(_ServerHost, _Room, _Host) ->
 search_affiliation(_ServerHost, _Room, _Host, _Affiliation) ->
     {error, not_implemented}.
 
-register_online_room(Room, Host, Pid) ->
+register_online_room(_ServerHost, Room, Host, Pid) ->
     F = fun() ->
 		mnesia:write(
 		  #muc_online_room{name_host = {Room, Host}, pid = Pid})
 	end,
     mnesia:transaction(F).
 
-unregister_online_room(Room, Host, Pid) ->
+unregister_online_room(_ServerHost, Room, Host, Pid) ->
     F = fun () ->
 		mnesia:delete_object(
 		  #muc_online_room{name_host = {Room, Host}, pid = Pid})
 	end,
     mnesia:transaction(F).
 
+find_online_room(_ServerHost, Room, Host) ->
+    find_online_room(Room, Host).
+
 find_online_room(Room, Host) ->
     case mnesia:dirty_read(muc_online_room, {Room, Host}) of
 	[] -> error;
 	[#muc_online_room{pid = Pid}] -> {ok, Pid}
     end.
 
-count_online_rooms(Host) ->
+count_online_rooms(_ServerHost, Host) ->
     ets:select_count(
       muc_online_room,
       ets:fun2ms(
@@ -185,20 +188,20 @@ count_online_rooms(Host) ->
 		H == Host
 	end)).
 
-get_online_rooms(Host,
+get_online_rooms(_ServerHost, Host,
 		 #rsm_set{max = Max, 'after' = After, before = undefined})
   when is_binary(After), After /= <<"">> ->
     lists:reverse(get_online_rooms(next, {After, Host}, Host, 0, Max, []));
-get_online_rooms(Host,
+get_online_rooms(_ServerHost, Host,
 		 #rsm_set{max = Max, 'after' = undefined, before = Before})
   when is_binary(Before), Before /= <<"">> ->
     get_online_rooms(prev, {Before, Host}, Host, 0, Max, []);
-get_online_rooms(Host,
+get_online_rooms(_ServerHost, Host,
 		 #rsm_set{max = Max, 'after' = undefined, before = <<"">>}) ->
     get_online_rooms(last, {<<"">>, Host}, Host, 0, Max, []);
-get_online_rooms(Host, #rsm_set{max = Max}) ->
+get_online_rooms(_ServerHost, Host, #rsm_set{max = Max}) ->
     lists:reverse(get_online_rooms(first, {<<"">>, Host}, Host, 0, Max, []));
-get_online_rooms(Host, undefined) ->
+get_online_rooms(_ServerHost, Host, undefined) ->
     mnesia:dirty_select(
       muc_online_room,
       ets:fun2ms(
@@ -248,17 +251,17 @@ get_online_rooms(Action, Key, Host, Count, Max, Items) ->
 rsm_supported() ->
     true.
 
-register_online_user({U, S, R}, Room, Host) ->
+register_online_user(_ServerHost, {U, S, R}, Room, Host) ->
     ets:insert(muc_online_users,
 	       #muc_online_users{us = {U, S}, resource = R,
 				 room = Room, host = Host}).
 
-unregister_online_user({U, S, R}, Room, Host) ->
+unregister_online_user(_ServerHost, {U, S, R}, Room, Host) ->
     ets:delete_object(muc_online_users,
 		      #muc_online_users{us = {U, S}, resource = R,
 					room = Room, host = Host}).
 
-count_online_rooms_by_user(U, S) ->
+count_online_rooms_by_user(_ServerHost, U, S) ->
     ets:select_count(
       muc_online_users,
       ets:fun2ms(
@@ -266,7 +269,7 @@ count_online_rooms_by_user(U, S) ->
 		U == U1 andalso S == S1
 	end)).
 
-get_online_rooms_by_user(U, S) ->
+get_online_rooms_by_user(_ServerHost, U, S) ->
     ets:select(
       muc_online_users,
       ets:fun2ms(

src/mod_muc_riak.erl

@@ -30,10 +30,10 @@
 %% API
 -export([init/2, import/3, store_room/4, restore_room/3, forget_room/3,
 	 can_use_nick/4, get_rooms/2, get_nick/3, set_nick/4]).
--export([register_online_room/3, unregister_online_room/3, find_online_room/2,
-	 get_online_rooms/2, count_online_rooms/1, rsm_supported/0,
-	 register_online_user/3, unregister_online_user/3,
-	 count_online_rooms_by_user/2, get_online_rooms_by_user/2]).
+-export([register_online_room/4, unregister_online_room/4, find_online_room/3,
+	 get_online_rooms/3, count_online_rooms/2, rsm_supported/0,
+	 register_online_user/4, unregister_online_user/4,
+	 count_online_rooms_by_user/3, get_online_rooms_by_user/3]).
 -export([set_affiliation/6, set_affiliations/4, get_affiliation/5,
 	 get_affiliations/3, search_affiliation/4]).
 
@@ -140,34 +140,34 @@ get_affiliations(_ServerHost, _Room, _Host) ->
 search_affiliation(_ServerHost, _Room, _Host, _Affiliation) ->
     {error, not_implemented}.
 
-register_online_room(_, _, _) ->
+register_online_room(_, _, _, _) ->
     erlang:error(not_implemented).
 
-unregister_online_room(_, _, _) ->
+unregister_online_room(_, _, _, _) ->
     erlang:error(not_implemented).
 
-find_online_room(_, _) ->
+find_online_room(_, _, _) ->
     erlang:error(not_implemented).
 
-count_online_rooms(_) ->
+count_online_rooms(_, _) ->
     erlang:error(not_implemented).
 
-get_online_rooms(_, _) ->
+get_online_rooms(_, _, _) ->
     erlang:error(not_implemented).
 
 rsm_supported() ->
     false.
 
-register_online_user(_, _, _) ->
+register_online_user(_, _, _, _) ->
     erlang:error(not_implemented).
 
-unregister_online_user(_, _, _) ->
+unregister_online_user(_, _, _, _) ->
     erlang:error(not_implemented).
 
-count_online_rooms_by_user(_, _) ->
+count_online_rooms_by_user(_, _, _) ->
     erlang:error(not_implemented).
 
-get_online_rooms_by_user(_, _) ->
+get_online_rooms_by_user(_, _, _) ->
     erlang:error(not_implemented).
 
 import(_LServer, <<"muc_room">>,

src/mod_muc_room.erl

@@ -30,10 +30,10 @@
 -behaviour(gen_fsm).
 
 %% External exports
--export([start_link/9,
-	 start_link/7,
-	 start/9,
-	 start/7,
+-export([start_link/10,
+	 start_link/8,
+	 start/10,
+	 start/8,
 	 get_role/2,
 	 get_affiliation/2,
 	 is_occupant_or_admin/2,
@@ -93,25 +93,25 @@
 %%% API
 %%%----------------------------------------------------------------------
 start(Host, ServerHost, Access, Room, HistorySize, RoomShaper,
-      Creator, Nick, DefRoomOpts) ->
+      Creator, Nick, DefRoomOpts, QueueType) ->
     gen_fsm:start(?MODULE, [Host, ServerHost, Access, Room, HistorySize,
-			    RoomShaper, Creator, Nick, DefRoomOpts],
+			    RoomShaper, Creator, Nick, DefRoomOpts, QueueType],
 		    ?FSMOPTS).
 
-start(Host, ServerHost, Access, Room, HistorySize, RoomShaper, Opts) ->
+start(Host, ServerHost, Access, Room, HistorySize, RoomShaper, Opts, QueueType) ->
     gen_fsm:start(?MODULE, [Host, ServerHost, Access, Room, HistorySize,
-			    RoomShaper, Opts],
+			    RoomShaper, Opts, QueueType],
 		    ?FSMOPTS).
 
 start_link(Host, ServerHost, Access, Room, HistorySize, RoomShaper,
-	   Creator, Nick, DefRoomOpts) ->
+	   Creator, Nick, DefRoomOpts, QueueType) ->
     gen_fsm:start_link(?MODULE, [Host, ServerHost, Access, Room, HistorySize,
-				 RoomShaper, Creator, Nick, DefRoomOpts],
+				 RoomShaper, Creator, Nick, DefRoomOpts, QueueType],
 		       ?FSMOPTS).
 
-start_link(Host, ServerHost, Access, Room, HistorySize, RoomShaper, Opts) ->
+start_link(Host, ServerHost, Access, Room, HistorySize, RoomShaper, Opts, QueueType) ->
     gen_fsm:start_link(?MODULE, [Host, ServerHost, Access, Room, HistorySize,
-				 RoomShaper, Opts],
+				 RoomShaper, Opts, QueueType],
 		       ?FSMOPTS).
 
 %%%----------------------------------------------------------------------
@@ -119,15 +119,17 @@ start_link(Host, ServerHost, Access, Room, HistorySize, RoomShaper, Opts) ->
 %%%----------------------------------------------------------------------
 
 init([Host, ServerHost, Access, Room, HistorySize,
-      RoomShaper, Creator, _Nick, DefRoomOpts]) ->
+      RoomShaper, Creator, _Nick, DefRoomOpts, QueueType]) ->
     process_flag(trap_exit, true),
     Shaper = shaper:new(RoomShaper),
+    RoomQueue = room_queue_new(ServerHost, Shaper, QueueType),
     State = set_affiliation(Creator, owner,
 	    #state{host = Host, server_host = ServerHost,
 		   access = Access, room = Room,
-		   history = lqueue_new(HistorySize),
+		   history = lqueue_new(HistorySize, QueueType),
 		   jid = jid:make(Room, Host),
 		   just_created = true,
+		   room_queue = RoomQueue,
 		   room_shaper = Shaper}),
     State1 = set_opts(DefRoomOpts, State),
     store_room(State1),
@@ -136,15 +138,17 @@ init([Host, ServerHost, Access, Room, HistorySize,
     add_to_log(room_existence, created, State1),
     add_to_log(room_existence, started, State1),
     {ok, normal_state, State1};
-init([Host, ServerHost, Access, Room, HistorySize, RoomShaper, Opts]) ->
+init([Host, ServerHost, Access, Room, HistorySize, RoomShaper, Opts, QueueType]) ->
     process_flag(trap_exit, true),
     Shaper = shaper:new(RoomShaper),
+    RoomQueue = room_queue_new(ServerHost, Shaper, QueueType),
     State = set_opts(Opts, #state{host = Host,
 				  server_host = ServerHost,
 				  access = Access,
 				  room = Room,
-				  history = lqueue_new(HistorySize),
+				  history = lqueue_new(HistorySize, QueueType),
 				  jid = jid:make(Room, Host),
+				  room_queue = RoomQueue,
 				  room_shaper = Shaper}),
     add_to_log(room_existence, started, State),
     {ok, normal_state, State}.
@@ -175,7 +179,10 @@ normal_state({route, <<"">>,
 	       MessageShaperInterval == 0 ->
 		    {RoomShaper, RoomShaperInterval} =
 			shaper:update(StateData#state.room_shaper, Size),
-		    RoomQueueEmpty = queue:is_empty(StateData#state.room_queue),
+		    RoomQueueEmpty = case StateData#state.room_queue of
+					 undefined -> true;
+					 RQ -> p1_queue:is_empty(RQ)
+				     end,
 		    if RoomShaperInterval == 0, RoomQueueEmpty ->
 			    NewActivity = Activity#activity{
 					    message_time = Now,
@@ -200,8 +207,8 @@ normal_state({route, <<"">>,
 					    message_time = Now,
 					    message_shaper = MessageShaper,
 					    message = Packet},
-			    RoomQueue = queue:in({message, From},
-						 StateData#state.room_queue),
+			    RoomQueue = p1_queue:in({message, From},
+						    StateData#state.room_queue),
 			    StateData2 = store_user_activity(From,
 							     NewActivity,
 							     StateData1),
@@ -584,8 +591,8 @@ code_change(_OldVsn, StateName, StateData, _Extra) ->
     {ok, StateName, StateData}.
 
 handle_info({process_user_presence, From}, normal_state = _StateName, StateData) ->
-    RoomQueueEmpty = queue:is_empty(StateData#state.room_queue),
-    RoomQueue = queue:in({presence, From}, StateData#state.room_queue),
+    RoomQueueEmpty = p1_queue:is_empty(StateData#state.room_queue),
+    RoomQueue = p1_queue:in({presence, From}, StateData#state.room_queue),
     StateData1 = StateData#state{room_queue = RoomQueue},
     if RoomQueueEmpty ->
 	   StateData2 = prepare_room_queue(StateData1),
@@ -595,9 +602,9 @@ handle_info({process_user_presence, From}, normal_state = _StateName, StateData)
 handle_info({process_user_message, From},
 	    normal_state = _StateName, StateData) ->
     RoomQueueEmpty =
-	queue:is_empty(StateData#state.room_queue),
-    RoomQueue = queue:in({message, From},
-			 StateData#state.room_queue),
+	p1_queue:is_empty(StateData#state.room_queue),
+    RoomQueue = p1_queue:in({message, From},
+			    StateData#state.room_queue),
     StateData1 = StateData#state{room_queue = RoomQueue},
     if RoomQueueEmpty ->
 	   StateData2 = prepare_room_queue(StateData1),
@@ -606,7 +613,7 @@ handle_info({process_user_message, From},
     end;
 handle_info(process_room_queue,
 	    normal_state = StateName, StateData) ->
-    case queue:out(StateData#state.room_queue) of
+    case p1_queue:out(StateData#state.room_queue) of
       {{value, {message, From}}, RoomQueue} ->
 	  Activity = get_user_activity(From, StateData),
 	  Packet = Activity#activity.message,
@@ -1149,13 +1156,13 @@ handle_iq_vcard(ToJID, NewId, #iq{type = Type, sub_els = SubEls} = IQ) ->
 -spec stanzaid_pack(binary(), binary()) -> binary().
 stanzaid_pack(OriginalId, Resource) ->
     <<"berd",
-      (jlib:encode_base64(<<"ejab\000",
+      (misc:encode_base64(<<"ejab\000",
 		       OriginalId/binary, "\000",
 		       Resource/binary>>))/binary>>.
 
 -spec stanzaid_unpack(binary()) -> {binary(), binary()}.
 stanzaid_unpack(<<"berd", StanzaIdBase64/binary>>) ->
-    StanzaId = jlib:decode_base64(StanzaIdBase64),
+    StanzaId = misc:decode_base64(StanzaIdBase64),
     [<<"ejab">>, OriginalId, Resource] =
 	str:tokens(StanzaId, <<"\000">>),
     {OriginalId, Resource}.
@@ -1418,6 +1425,32 @@ get_max_users_admin_threshold(StateData) ->
                            fun(I) when is_integer(I), I>0 -> I end,
                            5).
 
+-spec room_queue_new(binary(), shaper:shaper(), _) -> p1_queue:queue().
+room_queue_new(ServerHost, Shaper, QueueType) ->
+    HaveRoomShaper = Shaper /= none,
+    HaveMessageShaper = gen_mod:get_module_opt(
+			  ServerHost, mod_muc, user_message_shaper,
+			  fun(A) when is_atom(A) -> A end,
+			  none) /= none,
+    HavePresenceShaper = gen_mod:get_module_opt(
+			   ServerHost, mod_muc, user_presence_shaper,
+			   fun(A) when is_atom(A) -> A end,
+			   none) /= none,
+    HaveMinMessageInterval = gen_mod:get_module_opt(
+			       ServerHost, mod_muc, min_message_interval,
+			       fun(I) when is_number(I), I>=0 -> I end,
+			       0) /= 0,
+    HaveMinPresenceInterval = gen_mod:get_module_opt(
+				ServerHost, mod_muc, min_presence_interval,
+				fun(I) when is_number(I), I>=0 -> I end,
+				0) /= 0,
+    if HaveRoomShaper or HaveMessageShaper or HavePresenceShaper
+       or HaveMinMessageInterval or HaveMinPresenceInterval ->
+	    p1_queue:new(QueueType);
+       true ->
+	    undefined
+    end.
+
 -spec get_user_activity(jid(), state()) -> #activity{}.
 get_user_activity(JID, StateData) ->
     case treap:lookup(jid:tolower(JID),
@@ -1515,7 +1548,7 @@ clean_treap(Treap, CleanPriority) ->
 
 -spec prepare_room_queue(state()) -> state().
 prepare_room_queue(StateData) ->
-    case queue:out(StateData#state.room_queue) of
+    case p1_queue:out(StateData#state.room_queue) of
       {{value, {message, From}}, _RoomQueue} ->
 	  Activity = get_user_activity(From, StateData),
 	  Packet = Activity#activity.message,
@@ -1997,38 +2030,34 @@ get_history(Nick, Packet, #state{history = History}) ->
 	#muc{history = #muc_history{} = MUCHistory} ->
 	    Now = p1_time_compat:timestamp(),
 	    Q = History#lqueue.queue,
-	    {NewQ, Len} = filter_history(Q, MUCHistory, Now, Nick, queue:new(), 0, 0),
-	    History#lqueue{queue = NewQ, len = Len};
+	    filter_history(Q, Now, Nick, MUCHistory);
 	_ ->
-	    History
+	    p1_queue:to_list(History#lqueue.queue)
     end.
 
--spec filter_history(?TQUEUE, muc_history(), erlang:timestamp(), binary(),
-		     ?TQUEUE, non_neg_integer(), non_neg_integer()) ->
-			    {?TQUEUE, non_neg_integer()}.
-filter_history(Queue, #muc_history{since = Since,
-				   seconds = Seconds,
-				   maxstanzas = MaxStanzas,
-				   maxchars = MaxChars} = MUC,
-	       Now, Nick, AccQueue, NumStanzas, NumChars) ->
-    case queue:out_r(Queue) of
-	{{value, {_, _, _, TimeStamp, Size} = Elem}, NewQueue} ->
-	    NowDiff = timer:now_diff(Now, TimeStamp) div 1000000,
-	    Chars = Size + byte_size(Nick) + 1,
-	    if (NumStanzas < MaxStanzas) andalso
-	       (TimeStamp > Since) andalso
-	       (NowDiff =< Seconds) andalso
-	       (NumChars + Chars =< MaxChars) ->
-		    filter_history(NewQueue, MUC, Now, Nick,
-				   queue:in_r(Elem, AccQueue),
-				   NumStanzas + 1,
-				   NumChars + Chars);
-	       true ->
-		    {AccQueue, NumStanzas}
-	    end;
-	{empty, _} ->
-	    {AccQueue, NumStanzas}
-    end.
+-spec filter_history(p1_queue:queue(), erlang:timestamp(),
+		     binary(), muc_history()) -> list().
+filter_history(Queue, Now, Nick,
+	       #muc_history{since = Since,
+			    seconds = Seconds,
+			    maxstanzas = MaxStanzas,
+			    maxchars = MaxChars}) ->
+    {History, _, _} =
+	lists:foldr(
+	  fun({_, _, _, TimeStamp, Size} = Elem,
+	      {Elems, NumStanzas, NumChars} = Acc) ->
+		  NowDiff = timer:now_diff(Now, TimeStamp) div 1000000,
+		  Chars = Size + byte_size(Nick) + 1,
+		  if (NumStanzas < MaxStanzas) andalso
+		     (TimeStamp > Since) andalso
+		     (NowDiff =< Seconds) andalso
+		     (NumChars + Chars =< MaxChars) ->
+			  {[Elem|Elems], NumStanzas + 1, NumChars + Chars};
+		     true ->
+			  Acc
+		  end
+	  end, {[], 0, 0}, p1_queue:to_list(Queue)),
+    History.
 
 -spec is_room_overcrowded(state()) -> boolean().
 is_room_overcrowded(StateData) ->
@@ -2381,31 +2410,28 @@ status_codes(IsInitialPresence, _IsSelfPresence = true, StateData) ->
     end;
 status_codes(_IsInitialPresence, _IsSelfPresence = false, _StateData) -> [].
 
--spec lqueue_new(non_neg_integer()) -> lqueue().
-lqueue_new(Max) ->
-    #lqueue{queue = queue:new(), len = 0, max = Max}.
+-spec lqueue_new(non_neg_integer(), ram | file) -> lqueue().
+lqueue_new(Max, Type) ->
+    #lqueue{queue = p1_queue:new(Type), max = Max}.
 
 -spec lqueue_in(term(), lqueue()) -> lqueue().
 %% If the message queue limit is set to 0, do not store messages.
 lqueue_in(_Item, LQ = #lqueue{max = 0}) -> LQ;
 %% Otherwise, rotate messages in the queue store.
-lqueue_in(Item,
-	  #lqueue{queue = Q1, len = Len, max = Max}) ->
-    Q2 = queue:in(Item, Q1),
+lqueue_in(Item, #lqueue{queue = Q1, max = Max}) ->
+    Len = p1_queue:len(Q1),
+    Q2 = p1_queue:in(Item, Q1),
     if Len >= Max ->
 	   Q3 = lqueue_cut(Q2, Len - Max + 1),
-	   #lqueue{queue = Q3, len = Max, max = Max};
-       true -> #lqueue{queue = Q2, len = Len + 1, max = Max}
+	   #lqueue{queue = Q3, max = Max};
+       true -> #lqueue{queue = Q2, max = Max}
     end.
 
--spec lqueue_cut(queue:queue(), non_neg_integer()) -> queue:queue().
+-spec lqueue_cut(p1_queue:queue(), non_neg_integer()) -> p1_queue:queue().
 lqueue_cut(Q, 0) -> Q;
 lqueue_cut(Q, N) ->
-    {_, Q1} = queue:out(Q), lqueue_cut(Q1, N - 1).
-
--spec lqueue_to_list(lqueue()) -> list().
-lqueue_to_list(#lqueue{queue = Q1}) ->
-    queue:to_list(Q1).
+    {_, Q1} = p1_queue:out(Q),
+    lqueue_cut(Q1, N - 1).
 
 -spec add_message_to_history(binary(), jid(), message(), state()) -> state().
 add_message_to_history(FromNick, FromJID, Packet, StateData) ->
@@ -2436,7 +2462,7 @@ add_message_to_history(FromNick, FromJID, Packet, StateData) ->
 	    StateData
     end.
 
--spec send_history(jid(), lqueue(), state()) -> ok.
+-spec send_history(jid(), list(), state()) -> ok.
 send_history(JID, History, StateData) ->
     lists:foreach(
       fun({Nick, Packet, _HaveSubject, _TimeStamp, _Size}) ->
@@ -2445,7 +2471,7 @@ send_history(JID, History, StateData) ->
 		  Packet,
 		  jid:replace_resource(StateData#state.jid, Nick),
 		  JID))
-      end, lqueue_to_list(History)).
+      end, History).
 
 -spec send_subject(jid(), state()) -> ok.
 send_subject(JID, #state{subject_author = Nick} = StateData) ->

src/mod_muc_sql.erl

@@ -33,10 +33,10 @@
 -export([init/2, store_room/4, restore_room/3, forget_room/3,
 	 can_use_nick/4, get_rooms/2, get_nick/3, set_nick/4,
 	 import/3, export/1]).
--export([register_online_room/3, unregister_online_room/3, find_online_room/2,
-	 get_online_rooms/2, count_online_rooms/1, rsm_supported/0,
-	 register_online_user/3, unregister_online_user/3,
-	 count_online_rooms_by_user/2, get_online_rooms_by_user/2]).
+-export([register_online_room/4, unregister_online_room/4, find_online_room/3,
+	 get_online_rooms/3, count_online_rooms/2, rsm_supported/0,
+	 register_online_user/4, unregister_online_user/4,
+	 count_online_rooms_by_user/3, get_online_rooms_by_user/3]).
 -export([set_affiliation/6, set_affiliations/4, get_affiliation/5,
 	 get_affiliations/3, search_affiliation/4]).
 
@@ -48,11 +48,16 @@
 %%%===================================================================
 %%% API
 %%%===================================================================
-init(_Host, _Opts) ->
-    ok.
+init(Host, Opts) ->
+    case gen_mod:ram_db_mod(Host, Opts, mod_muc) of
+	?MODULE ->
+	    clean_tables(Host);
+	_ ->
+	    ok
+    end.
 
 store_room(LServer, Host, Name, Opts) ->
-    SOpts = jlib:term_to_expr(Opts),
+    SOpts = misc:term_to_expr(Opts),
     F = fun () ->
 		?SQL_UPSERT_T(
                    "muc_room",
@@ -165,42 +170,133 @@ get_affiliations(_ServerHost, _Room, _Host) ->
 search_affiliation(_ServerHost, _Room, _Host, _Affiliation) ->
     {error, not_implemented}.
 
-register_online_room(_, _, _) ->
-    erlang:error(not_implemented).
+register_online_room(ServerHost, Room, Host, Pid) ->
+    PidS = misc:encode_pid(Pid),
+    NodeS = erlang:atom_to_binary(node(Pid), latin1),
+    case ?SQL_UPSERT(ServerHost,
+		     "muc_online_room",
+		     ["!name=%(Room)s",
+		      "!host=%(Host)s",
+		      "node=%(NodeS)s",
+		      "pid=%(PidS)s"]) of
+	ok ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to update 'muc_online_room': ~p", [Err]),
+	    Err
+    end.
 
-unregister_online_room(_, _, _) ->
-    erlang:error(not_implemented).
+unregister_online_room(ServerHost, Room, Host, Pid) ->
+    %% TODO: report errors
+    PidS = misc:encode_pid(Pid),
+    NodeS = erlang:atom_to_binary(node(Pid), latin1),
+    ejabberd_sql:sql_query(
+      ServerHost,
+      ?SQL("delete from muc_online_room where name=%(Room)s and "
+	   "host=%(Host)s and node=%(NodeS)s and pid=%(PidS)s")).
 
-find_online_room(_, _) ->
-    erlang:error(not_implemented).
+find_online_room(ServerHost, Room, Host) ->
+    case ejabberd_sql:sql_query(
+	   ServerHost,
+	   ?SQL("select @(pid)s, @(node)s from muc_online_room where "
+		"name=%(Room)s and host=%(Host)s")) of
+	{selected, [{PidS, NodeS}]} ->
+	    try {ok, misc:decode_pid(PidS, NodeS)}
+	    catch _:{bad_node, _} -> error
+	    end;
+	{selected, []} ->
+	    error;
+	Err ->
+	    ?ERROR_MSG("failed to select 'muc_online_room': ~p", [Err]),
+	    error
+    end.
 
-count_online_rooms(_) ->
-    erlang:error(not_implemented).
+count_online_rooms(ServerHost, Host) ->
+    case ejabberd_sql:sql_query(
+	   ServerHost,
+	   ?SQL("select @(count(*))d from muc_online_room "
+		"where host=%(Host)s")) of
+	{selected, [{Num}]} ->
+	    Num;
+	Err ->
+	    ?ERROR_MSG("failed to select 'muc_online_room': ~p", [Err]),
+	    0
+    end.
 
-get_online_rooms(_, _) ->
-    erlang:error(not_implemented).
+get_online_rooms(ServerHost, Host, _RSM) ->
+    case ejabberd_sql:sql_query(
+	   ServerHost,
+	   ?SQL("select @(name)s, @(pid)s, @(node)s from muc_online_room "
+		"where host=%(Host)s")) of
+	{selected, Rows} ->
+	    lists:flatmap(
+	      fun({Room, PidS, NodeS}) ->
+		      try [{Room, Host, misc:decode_pid(PidS, NodeS)}]
+		      catch _:{bad_node, _} -> []
+		      end
+	      end, Rows);
+	Err ->
+	    ?ERROR_MSG("failed to select 'muc_online_room': ~p", [Err]),
+	    []
+    end.
 
 rsm_supported() ->
     false.
 
-register_online_user(_, _, _) ->
-    erlang:error(not_implemented).
+register_online_user(ServerHost, {U, S, R}, Room, Host) ->
+    NodeS = erlang:atom_to_binary(node(), latin1),
+    case ?SQL_UPSERT(ServerHost, "muc_online_users",
+		     ["!username=%(U)s",
+		      "!server=%(S)s",
+		      "!resource=%(R)s",
+		      "!name=%(Room)s",
+		      "!host=%(Host)s",
+		      "node=%(NodeS)s"]) of
+	ok ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to update 'muc_online_users': ~p", [Err]),
+	    Err
+    end.
 
-unregister_online_user(_, _, _) ->
-    erlang:error(not_implemented).
+unregister_online_user(ServerHost, {U, S, R}, Room, Host) ->
+    %% TODO: report errors
+    ejabberd_sql:sql_query(
+      ServerHost,
+      ?SQL("delete from muc_online_users where username=%(U)s and "
+	   "server=%(S)s and resource=%(R)s and name=%(Room)s and "
+	   "host=%(Host)s")).
 
-count_online_rooms_by_user(_, _) ->
-    erlang:error(not_implemented).
+count_online_rooms_by_user(ServerHost, U, S) ->
+    case ejabberd_sql:sql_query(
+	   ServerHost,
+	   ?SQL("select @(count(*))d from muc_online_users where "
+		"username=%(U)s and server=%(S)s")) of
+	{selected, [{Num}]} ->
+	    Num;
+	Err ->
+	    ?ERROR_MSG("failed to select 'muc_online_users': ~p", [Err]),
+	    0
+    end.
 
-get_online_rooms_by_user(_, _) ->
-    erlang:error(not_implemented).
+get_online_rooms_by_user(ServerHost, U, S) ->
+    case ejabberd_sql:sql_query(
+	   ServerHost,
+	   ?SQL("select @(name)s, @(host)s from muc_online_users where "
+		"username=%(U)s and server=%(S)s")) of
+	{selected, Rows} ->
+	    Rows;
+	Err ->
+	    ?ERROR_MSG("failed to select 'muc_online_users': ~p", [Err]),
+	    []
+    end.
 
 export(_Server) ->
     [{muc_room,
       fun(Host, #muc_room{name_host = {Name, RoomHost}, opts = Opts}) ->
               case str:suffix(Host, RoomHost) of
                   true ->
-                      SOpts = jlib:term_to_expr(Opts),
+                      SOpts = misc:term_to_expr(Opts),
                       [?SQL("delete from muc_room where name=%(Name)s"
                             " and host=%(RoomHost)s;"),
                        ?SQL("insert into muc_room(name, host, opts) "
@@ -232,3 +328,25 @@ import(_, _, _) ->
 %%%===================================================================
 %%% Internal functions
 %%%===================================================================
+clean_tables(ServerHost) ->
+    NodeS = erlang:atom_to_binary(node(), latin1),
+    ?INFO_MSG("Cleaning SQL muc_online_room table...", []),
+    case ejabberd_sql:sql_query(
+	   ServerHost,
+	   ?SQL("delete from muc_online_room where node=%(NodeS)s")) of
+	{updated, _} ->
+	    ok;
+	Err1 ->
+	    ?ERROR_MSG("failed to clean 'muc_online_room' table: ~p", [Err1]),
+	    Err1
+    end,
+    ?INFO_MSG("Cleaning SQL muc_online_users table...", []),
+    case ejabberd_sql:sql_query(
+	   ServerHost,
+	   ?SQL("delete from muc_online_users where node=%(NodeS)s")) of
+	{updated, _} ->
+	    ok;
+	Err2 ->
+	    ?ERROR_MSG("failed to clean 'muc_online_users' table: ~p", [Err2]),
+	    Err2
+    end.

src/mod_ping.erl

@@ -146,12 +146,14 @@ handle_cast({iq_pong, JID, timeout}, State) ->
 	      JID,
 	  case ejabberd_sm:get_session_pid(User, Server, Resource)
 	      of
-	    Pid when is_pid(Pid) -> ejabberd_c2s:close(Pid);
+	    Pid when is_pid(Pid) -> ejabberd_c2s:close(Pid, _SendTrailer = false);
 	    _ -> ok
 	  end;
       _ -> ok
     end,
     {noreply, State#state{timers = Timers}};
+handle_cast({iq_pong, _JID, _}, State) ->
+    {noreply, State};
 handle_cast(Msg, State) ->
     ?WARNING_MSG("unexpected cast: ~p", [Msg]),
     {noreply, State}.

src/mod_pres_counter.erl

@@ -113,7 +113,7 @@ update(Server, JID, Dir) ->
 		       ?WARNING_MSG("Flooder detected: ~s, on IP: ~s ignoring "
 				    "sent presence subscriptions~n",
 				    [jid:encode(JID),
-				     jlib:ip_to_list(IP)])
+				     misc:ip_to_list(IP)])
 		 end,
 		 {stop, deny};
 	     true ->

src/mod_proxy65_redis.erl

@@ -0,0 +1,186 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% Created : 31 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2017   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(mod_proxy65_redis).
+-behaviour(mod_proxy65).
+
+%% API
+-export([init/0, register_stream/2, unregister_stream/1, activate_stream/4]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+
+-record(proxy65, {pid_t :: pid(),
+		  pid_i :: pid() | undefined,
+		  jid_i :: binary() | undefined}).
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+init() ->
+    ?INFO_MSG("Cleaning Redis 'proxy65' table...", []),
+    NodeKey = node_key(),
+    case ejabberd_redis:smembers(NodeKey) of
+	{ok, SIDs} ->
+	    SIDKeys = [sid_key(S) || S <- SIDs],
+	    JIDs = lists:flatmap(
+		     fun(SIDKey) ->
+			     case ejabberd_redis:get(SIDKey) of
+				 {ok, Val} ->
+				     try binary_to_term(Val) of
+					 #proxy65{jid_i = J} when is_binary(J) ->
+					     [jid_key(J)];
+					 _ ->
+					     []
+				     catch _:badarg ->
+					     []
+				     end;
+				 _ ->
+				     []
+			     end
+		     end, SIDKeys),
+	    ejabberd_redis:multi(
+	      fun() ->
+		      if SIDs /= [] ->
+			      ejabberd_redis:del(SIDKeys),
+			      if JIDs /= [] ->
+				      ejabberd_redis:del(JIDs);
+				 true ->
+				      ok
+			      end;
+			 true ->
+			      ok
+		      end,
+		      ejabberd_redis:del([NodeKey])
+	      end),
+	    ok;
+	{error, _} ->
+	    {error, db_failure}
+    end.
+
+register_stream(SID, Pid) ->
+    SIDKey = sid_key(SID),
+    try
+	{ok, Val} = ejabberd_redis:get(SIDKey),
+	try binary_to_term(Val) of
+	    #proxy65{pid_i = undefined} = R ->
+		NewVal = term_to_binary(R#proxy65{pid_i = Pid}),
+		ok = ejabberd_redis:set(SIDKey, NewVal);
+	    _ ->
+		{error, conflict}
+	catch _:badarg when Val == undefined ->
+		NewVal = term_to_binary(#proxy65{pid_t = Pid}),
+		{ok, _} = ejabberd_redis:multi(
+			    fun() ->
+				    ejabberd_redis:set(SIDKey, NewVal),
+				    ejabberd_redis:sadd(node_key(), [SID])
+			    end),
+		ok;
+	      _:badarg ->
+		?ERROR_MSG("malformed data in redis (key = '~s'): ~p",
+			   [SIDKey, Val]),
+		{error, db_failure}
+	end
+    catch _:{badmatch, {error, _}} ->
+	    {error, db_failure}
+    end.
+
+unregister_stream(SID) ->
+    SIDKey = sid_key(SID),
+    NodeKey = node_key(),
+    try
+	{ok, Val} = ejabberd_redis:get(SIDKey),
+	try binary_to_term(Val) of
+	    #proxy65{jid_i = JID} when is_binary(JID) ->
+		JIDKey = jid_key(JID),
+		{ok, _} = ejabberd_redis:multi(
+			    fun() ->
+				    ejabberd_redis:del([SIDKey]),
+				    ejabberd_redis:srem(JIDKey, [SID]),
+				    ejabberd_redis:srem(NodeKey, [SID])
+			    end),
+		ok;
+	    _ ->
+		{ok, _} = ejabberd_redis:multi(
+			    fun() ->
+				    ejabberd_redis:del([SIDKey]),
+				    ejabberd_redis:srem(NodeKey, [SID])
+			    end),
+		ok
+	catch _:badarg when Val == undefined ->
+		ok;
+	      _:badarg ->
+		?ERROR_MSG("malformed data in redis (key = '~s'): ~p",
+			   [SIDKey, Val]),
+		{error, db_failure}
+	end
+    catch _:{badmatch, {error, _}} ->
+	    {error, db_failure}
+    end.
+
+activate_stream(SID, IJID, MaxConnections, _Node) ->
+    SIDKey = sid_key(SID),
+    JIDKey = jid_key(IJID),
+    try
+	{ok, Val} = ejabberd_redis:get(SIDKey),
+	try binary_to_term(Val) of
+	    #proxy65{pid_t = TPid, pid_i = IPid,
+		     jid_i = undefined} = R when is_pid(IPid) ->
+		{ok, Num} = ejabberd_redis:scard(JIDKey),
+		if Num >= MaxConnections ->
+			{error, {limit, IPid, TPid}};
+		   true ->
+			NewVal = term_to_binary(R#proxy65{jid_i = IJID}),
+			{ok, _} = ejabberd_redis:multi(
+				    fun() ->
+					    ejabberd_redis:sadd(JIDKey, [SID]),
+					    ejabberd_redis:set(SIDKey, NewVal)
+				    end),
+			{ok, IPid, TPid}
+		end;
+	    #proxy65{jid_i = JID} when is_binary(JID) ->
+		{error, conflict};
+	    _ ->
+		{error, notfound}
+	catch _:badarg when Val == undefined ->
+		{error, notfound};
+	      _:badarg ->
+		?ERROR_MSG("malformed data in redis (key = '~s'): ~p",
+			   [SIDKey, Val]),
+		{error, db_failure}
+	end
+    catch _:{badmatch, {error, _}} ->
+	    {error, db_failure}
+    end.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================
+sid_key(SID) ->
+    <<"ejabberd:proxy65:sid:", SID/binary>>.
+
+jid_key(JID) ->
+    <<"ejabberd:proxy65:initiator:", JID/binary>>.
+
+node_key() ->
+    Node = erlang:atom_to_binary(node(), latin1),
+    <<"ejabberd:proxy65:node:", Node/binary>>.

src/mod_proxy65_service.erl

@@ -253,9 +253,9 @@ process_bytestreams(#iq{type = set, lang = Lang, from = InitiatorJID, to = To,
 transform_module_options(Opts) ->
     lists:map(
       fun({ip, IP}) when is_tuple(IP) ->
-              {ip, jlib:ip_to_list(IP)};
+              {ip, misc:ip_to_list(IP)};
          ({hostname, IP}) when is_tuple(IP) ->
-              {hostname, jlib:ip_to_list(IP)};
+              {hostname, misc:ip_to_list(IP)};
          (Opt) ->
               Opt
       end, Opts).
@@ -265,7 +265,7 @@ get_streamhost(Host, ServerHost) ->
     {Port, IP} = get_port_ip(ServerHost),
     HostName = gen_mod:get_module_opt(ServerHost, mod_proxy65, hostname,
 				      fun iolist_to_binary/1,
-				      jlib:ip_to_list(IP)),
+				      misc:ip_to_list(IP)),
     Resource = ejabberd_cluster:node_id(),
     #streamhost{jid = jid:make(<<"">>, Host, Resource),
 		host = HostName,

src/mod_proxy65_sql.erl

@@ -0,0 +1,142 @@
+%%%-------------------------------------------------------------------
+%%% @author Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%% Created : 30 Mar 2017 by Evgeny Khramtsov <ekhramtsov@process-one.net>
+%%%
+%%%
+%%% ejabberd, Copyright (C) 2002-2017   ProcessOne
+%%%
+%%% This program is free software; you can redistribute it and/or
+%%% modify it under the terms of the GNU General Public License as
+%%% published by the Free Software Foundation; either version 2 of the
+%%% License, or (at your option) any later version.
+%%%
+%%% This program is distributed in the hope that it will be useful,
+%%% but WITHOUT ANY WARRANTY; without even the implied warranty of
+%%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+%%% General Public License for more details.
+%%%
+%%% You should have received a copy of the GNU General Public License along
+%%% with this program; if not, write to the Free Software Foundation, Inc.,
+%%% 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+%%%
+%%%-------------------------------------------------------------------
+-module(mod_proxy65_sql).
+-behaviour(mod_proxy65).
+
+-compile([{parse_transform, ejabberd_sql_pt}]).
+
+%% API
+-export([init/0, register_stream/2, unregister_stream/1, activate_stream/4]).
+
+-include("ejabberd.hrl").
+-include("logger.hrl").
+-include("ejabberd_sql_pt.hrl").
+
+%%%===================================================================
+%%% API
+%%%===================================================================
+init() ->
+    NodeS = erlang:atom_to_binary(node(), latin1),
+    ?INFO_MSG("Cleaning SQL 'proxy65' table...", []),
+    case ejabberd_sql:sql_query(
+	   ?MYNAME,
+	   ?SQL("delete from proxy65 where "
+		"node_i=%(NodeS)s or node_t=%(NodeS)s")) of
+	{updated, _} ->
+	    ok;
+	Err ->
+	    ?ERROR_MSG("failed to clean 'proxy65' table: ~p", [Err]),
+	    Err
+    end.
+
+register_stream(SID, Pid) ->
+    PidS = misc:encode_pid(Pid),
+    NodeS = erlang:atom_to_binary(node(Pid), latin1),
+    F = fun() ->
+		case ejabberd_sql:sql_query_t(
+		       ?SQL("update proxy65 set pid_i=%(PidS)s, "
+			    "node_i=%(NodeS)s where sid=%(SID)s")) of
+		    {updated, 1} ->
+			ok;
+		    _ ->
+			ejabberd_sql:sql_query_t(
+			  ?SQL("insert into proxy65"
+			       "(sid, pid_t, node_t, pid_i, node_i, jid_i) "
+			       "values (%(SID)s, %(PidS)s, %(NodeS)s, '', '', '')"))
+		end
+	end,
+    case ejabberd_sql:sql_transaction(?MYNAME, F) of
+	{atomic, _} ->
+	    ok;
+	{aborted, Reason} ->
+	    ?ERROR_MSG("failed to register stream: ~p", [Reason]),
+	    {error, Reason}
+    end.
+
+unregister_stream(SID) ->
+    F = fun() ->
+		ejabberd_sql:sql_query_t(
+		  ?SQL("delete from proxy65 where sid=%(SID)s"))
+	end,
+    case ejabberd_sql:sql_transaction(?MYNAME, F) of
+	{atomic, _} ->
+	    ok;
+	{aborted, Reason} ->
+	    ?ERROR_MSG("failed to unregister stream: ~p", [Reason]),
+	    {error, Reason}
+    end.
+
+activate_stream(SID, IJID, MaxConnections, _Node) ->
+    F = fun() ->
+		case ejabberd_sql:sql_query_t(
+		       ?SQL("select @(pid_t)s, @(node_t)s, @(pid_i)s, "
+			    "@(node_i)s, @(jid_i)s from proxy65 where "
+			    "sid=%(SID)s")) of
+		    {selected, [{TPidS, TNodeS, IPidS, INodeS, <<"">>}]}
+		      when IPidS /= <<"">> ->
+			try {misc:decode_pid(TPidS, TNodeS),
+			     misc:decode_pid(IPidS, INodeS)} of
+			    {TPid, IPid} ->
+				case ejabberd_sql:sql_query_t(
+				       ?SQL("update proxy65 set jid_i=%(IJID)s "
+					    "where sid=%(SID)s")) of
+				    {updated, 1} when is_integer(MaxConnections) ->
+					case ejabberd_sql:sql_query_t(
+					       ?SQL("select @(count(*))d from proxy65 "
+						    "where jid_i=%(IJID)s")) of
+					    {selected, [{Num}]} when Num > MaxConnections ->
+						ejabberd_sql:abort({limit, IPid, TPid});
+					    {selected, _} ->
+						{ok, IPid, TPid};
+					    Err ->
+						ejabberd_sql:abort(Err)
+					end;
+				    {updated, _} ->
+					{ok, IPid, TPid};
+				    Err ->
+					ejabberd_sql:abort(Err)
+				end
+			catch _:{bad_node, _} ->
+				{error, notfound}
+			end;
+		    {selected, [{_, _, _, _, JID}]} when JID /= <<"">> ->
+			{error, conflict};
+		    {selected, _} ->
+			{error, notfound};
+		    Err ->
+			ejabberd_sql:abort(Err)
+		end
+	end,
+    case ejabberd_sql:sql_transaction(?MYNAME, F) of
+	{atomic, Result} ->
+	    Result;
+	{aborted, {limit, _, _} = Limit} ->
+	    {error, Limit};
+	{aborted, Reason} ->
+	    ?ERROR_MSG("failed to activate bytestream: ~p", [Reason]),
+	    {error, Reason}
+    end.
+
+%%%===================================================================
+%%% Internal functions
+%%%===================================================================

src/mod_pubsub.erl

@@ -152,6 +152,7 @@
 -type(pubsubState() ::
     #pubsub_state{
 	stateid       :: {Entity::ljid(), Nidx::mod_pubsub:nodeIdx()},
+	nodeidx       :: Nidx::mod_pubsub:nodeIdx(),
 	items         :: [ItemId::mod_pubsub:itemId()],
 	affiliation   :: Affs::mod_pubsub:affiliation(),
 	subscriptions :: [{Sub::mod_pubsub:subscription(), SubId::mod_pubsub:subId()}]
@@ -161,6 +162,7 @@
 -type(pubsubItem() ::
     #pubsub_item{
 	itemid       :: {ItemId::mod_pubsub:itemId(), Nidx::mod_pubsub:nodeIdx()},
+	nodeidx      :: Nidx::mod_pubsub:nodeIdx(),
 	creation     :: {erlang:timestamp(), ljid()},
 	modification :: {erlang:timestamp(), ljid()},
 	payload      :: mod_pubsub:payload()
@@ -255,7 +257,7 @@ init([ServerHost, Opts]) ->
     MaxSubsNode = gen_mod:get_opt(max_subscriptions_node, Opts,
 	    fun(A) when is_integer(A) andalso A >= 0 -> A end, undefined),
     case gen_mod:db_type(ServerHost, ?MODULE) of
-	mnesia -> init_mnesia(Host, ServerHost, Opts);
+	mnesia -> pubsub_index:init(Host, ServerHost, Opts);
 	_ -> ok
     end,
     {Plugins, NodeTree, PepMapping} = init_plugins(Host, ServerHost, Opts),
@@ -372,18 +374,6 @@ depends(ServerHost, Opts) ->
 	      end
       end, Plugins).
 
-init_mnesia(Host, ServerHost, Opts) ->
-    pubsub_index:init(Host, ServerHost, Opts),
-    spawn(fun() ->
-	      %% maybe upgrade db. this can take time when upgrading existing
-	      %% data from ejabberd 2.1.x, so we don't want this to block
-	      %% calling gen_server:start
-	      pubsub_migrate:update_node_database(Host, ServerHost),
-	      pubsub_migrate:update_state_database(Host, ServerHost),
-	      pubsub_migrate:update_item_database(Host, ServerHost),
-	      pubsub_migrate:update_lastitem_database(Host, ServerHost)
-	  end).
-
 %% @doc Call the init/1 function for each plugin declared in the config file.
 %% The default plugin module is implicit.
 %% <p>The Erlang code for the plugin is located in a module called
@@ -1215,7 +1205,7 @@ iq_pubsub(Host, Access, #iq{from = From, type = IQType, lang = Lang,
 		    create_node(Host, ServerHost, Node, From, Type, Access, Config)
 	    end;
 	{set, #pubsub{publish = #ps_publish{node = Node, items = Items},
-		      publish_options = XData, _ = undefined}} ->
+		      publish_options = XData, configure = _, _ = undefined}} ->
 	    ServerHost = serverhost(Host),
 	    case Items of
 		[#ps_item{id = ItemId, xml_els = Payload}] ->
@@ -2165,48 +2155,21 @@ get_allowed_items_call(Host, Nidx, From, Type, Options, Owners, RSM) ->
     {PS, RG} = get_presence_and_roster_permissions(Host, From, Owners, AccessModel, AllowedGroups),
     node_call(Host, Type, get_items, [Nidx, From, AccessModel, PS, RG, undefined, RSM]).
 
-get_last_item(Host, Type, Nidx, LJID) ->
-    case get_cached_item(Host, Nidx) of
-	undefined -> get_last_item(Host, Type, Nidx, LJID, gen_mod:db_type(serverhost(Host), ?MODULE));
-	LastItem -> LastItem
-    end.
-get_last_item(Host, Type, Nidx, LJID, mnesia) ->
-    case node_action(Host, Type, get_items, [Nidx, LJID, undefined]) of
-	{result, {[LastItem|_], _}} -> LastItem;
-	_ -> undefined
-    end;
-get_last_item(Host, Type, Nidx, LJID, sql) ->
-    case node_action(Host, Type, get_last_items, [Nidx, LJID, 1]) of
-	{result, [LastItem]} -> LastItem;
-	_ -> undefined
-    end;
-get_last_item(_Host, _Type, _Nidx, _LJID, _) ->
-    undefined.
-
-get_last_items(Host, Type, Nidx, LJID, Number) ->
-    get_last_items(Host, Type, Nidx, LJID, Number, gen_mod:db_type(serverhost(Host), ?MODULE)).
-get_last_items(Host, Type, Nidx, LJID, Number, mnesia) ->
-    case node_action(Host, Type, get_items, [Nidx, LJID, undefined]) of
-	{result, {Items, _}} -> lists:sublist(Items, Number);
-	_ -> []
-    end;
-get_last_items(Host, Type, Nidx, LJID, Number, sql) ->
-    case node_action(Host, Type, get_last_items, [Nidx, LJID, Number]) of
+get_last_items(Host, Type, Nidx, LJID, Count) ->
+    case node_action(Host, Type, get_last_items, [Nidx, LJID, Count]) of
 	{result, Items} -> Items;
 	_ -> []
-    end;
-get_last_items(_Host, _Type, _Nidx, _LJID, _Number, _) ->
-    [].
+    end.
 
 %% @doc <p>Resend the items of a node to the user.</p>
 %% @todo use cache-last-item feature
 send_items(Host, Node, Nidx, Type, Options, LJID, last) ->
-    case get_last_item(Host, Type, Nidx, LJID) of
-	undefined ->
-	    ok;
-	LastItem ->
+    case get_last_items(Host, Type, Nidx, LJID, 1) of
+	[LastItem] ->
 	    Stanza = items_event_stanza(Node, Options, [LastItem]),
-	    dispatch_items(Host, LJID, Node, Stanza)
+	    dispatch_items(Host, LJID, Node, Stanza);
+	_ ->
+	    ok
     end;
 send_items(Host, Node, Nidx, Type, Options, LJID, Number) when Number > 0 ->
     Stanza = items_event_stanza(Node, Options, get_last_items(Host, Type, Nidx, Number, LJID)),
@@ -2215,23 +2178,12 @@ send_items(Host, Node, _Nidx, _Type, Options, LJID, _) ->
     Stanza = items_event_stanza(Node, Options, []),
     dispatch_items(Host, LJID, Node, Stanza).
 
-dispatch_items({FromU, FromS, FromR} = From, {ToU, ToS, ToR} = To,
-	    Node, Stanza) ->
-    C2SPid = case ejabberd_sm:get_session_pid(ToU, ToS, ToR) of
-	ToPid when is_pid(ToPid) -> ToPid;
-	_ ->
-	    R = user_resource(FromU, FromS, FromR),
-	    case ejabberd_sm:get_session_pid(FromU, FromS, R) of
-		FromPid when is_pid(FromPid) -> FromPid;
-		_ -> undefined
-	    end
-    end,
-    if C2SPid == undefined -> ok;
-	true ->
-	    C2SPid ! {send_filtered, {pep_message, <<Node/binary, "+notify">>},
-		service_jid(From), jid:make(To),
-		      Stanza}
-    end;
+dispatch_items({FromU, FromS, FromR}, To, Node, Stanza) ->
+    SenderResource = user_resource(FromU, FromS, FromR),
+    ejabberd_sm:route(jid:make(FromU, FromS, SenderResource),
+		      {send_filtered, {pep_message, <<((Node))/binary, "+notify">>},
+		       jid:make(FromU, FromS), jid:make(To),
+		       Stanza});
 dispatch_items(From, To, _Node, Stanza) ->
     ejabberd_router:route(
       xmpp:set_from_to(Stanza, service_jid(From), jid:make(To))).
@@ -2577,28 +2529,21 @@ get_subscriptions(Host, Node, JID) ->
 	    Error
     end.
 
-get_subscriptions_for_send_last(Host, PType, mnesia, JID, LJID, BJID) ->
+get_subscriptions_for_send_last(Host, PType, sql, JID, LJID, BJID) ->
+    {result, Subs} = node_action(Host, PType,
+	    get_entity_subscriptions_for_send_last,
+	    [Host, JID]),
+    [{Node, Sub, SubId, SubJID}
+	|| {Node, Sub, SubId, SubJID} <- Subs,
+	    Sub =:= subscribed, (SubJID == LJID) or (SubJID == BJID)];
+get_subscriptions_for_send_last(Host, PType, _, JID, LJID, BJID) ->
     {result, Subs} = node_action(Host, PType,
 	    get_entity_subscriptions,
 	    [Host, JID]),
     [{Node, Sub, SubId, SubJID}
 	|| {Node, Sub, SubId, SubJID} <- Subs,
 	    Sub =:= subscribed, (SubJID == LJID) or (SubJID == BJID),
-	    match_option(Node, send_last_published_item, on_sub_and_presence)];
-get_subscriptions_for_send_last(Host, PType, sql, JID, LJID, BJID) ->
-    case catch node_action(Host, PType,
-	    get_entity_subscriptions_for_send_last,
-	    [Host, JID])
-    of
-	{result, Subs} ->
-	    [{Node, Sub, SubId, SubJID}
-		|| {Node, Sub, SubId, SubJID} <- Subs,
-		    Sub =:= subscribed, (SubJID == LJID) or (SubJID == BJID)];
-	_ ->
-	    []
-    end;
-get_subscriptions_for_send_last(_Host, _PType, _, _JID, _LJID, _BJID) ->
-    [].
+	    match_option(Node, send_last_published_item, on_sub_and_presence)].
 
 -spec set_subscriptions(host(), binary(), jid(), [ps_subscription()]) ->
 			       {result, undefined} | {error, stanza_error()}.
@@ -3016,11 +2961,11 @@ broadcast_stanza({LUser, LServer, LResource}, Publisher, Node, Nidx, Type, NodeO
     broadcast_stanza({LUser, LServer, <<>>}, Node, Nidx, Type, NodeOptions, SubsByDepth, NotifyType, BaseStanza, SHIM),
     %% Handles implicit presence subscriptions
     SenderResource = user_resource(LUser, LServer, LResource),
-	    NotificationType = get_option(NodeOptions, notification_type, headline),
-	    Stanza = add_message_type(BaseStanza, NotificationType),
-	    %% set the from address on the notification to the bare JID of the account owner
-	    %% Also, add "replyto" if entity has presence subscription to the account owner
-	    %% See XEP-0163 1.1 section 4.3.1
+    NotificationType = get_option(NodeOptions, notification_type, headline),
+    Stanza = add_message_type(BaseStanza, NotificationType),
+    %% set the from address on the notification to the bare JID of the account owner
+    %% Also, add "replyto" if entity has presence subscription to the account owner
+    %% See XEP-0163 1.1 section 4.3.1
     ejabberd_sm:route(jid:make(LUser, LServer, SenderResource),
 		      {pep_message, <<((Node))/binary, "+notify">>,
 		       jid:make(LUser, LServer),
@@ -3213,28 +3158,18 @@ filter_node_options(Options, BaseOptions) ->
 
 -spec node_owners_action(host(), binary(), nodeIdx(), [ljid()]) -> [ljid()].
 node_owners_action(Host, Type, Nidx, []) ->
-    case gen_mod:db_type(serverhost(Host), ?MODULE) of
-	sql ->
-	    case node_action(Host, Type, get_node_affiliations, [Nidx]) of
-		{result, Affs} -> [LJID || {LJID, Aff} <- Affs, Aff =:= owner];
-		_ -> []
-	    end;
-	_ ->
-	    []
+    case node_action(Host, Type, get_node_affiliations, [Nidx]) of
+	{result, Affs} -> [LJID || {LJID, Aff} <- Affs, Aff =:= owner];
+	_ -> []
     end;
 node_owners_action(_Host, _Type, _Nidx, Owners) ->
     Owners.
 
 -spec node_owners_call(host(), binary(), nodeIdx(), [ljid()]) -> [ljid()].
 node_owners_call(Host, Type, Nidx, []) ->
-    case gen_mod:db_type(serverhost(Host), ?MODULE) of
-	sql ->
-	    case node_call(Host, Type, get_node_affiliations, [Nidx]) of
-		{result, Affs} -> [LJID || {LJID, Aff} <- Affs, Aff =:= owner];
-		_ -> []
-	    end;
-	_ ->
-	    []
+    case node_call(Host, Type, get_node_affiliations, [Nidx]) of
+	{result, Affs} -> [LJID || {LJID, Aff} <- Affs, Aff =:= owner];
+	_ -> []
     end;
 node_owners_call(_Host, _Type, _Nidx, Owners) ->
     Owners.
@@ -3479,23 +3414,15 @@ tree(Host) ->
 	Tree -> Tree
     end.
 
--spec tree(host(), binary() | atom()) -> atom().
+-spec tree(host(), binary()) -> atom().
 tree(_Host, <<"virtual">>) ->
     nodetree_virtual;   % special case, virtual does not use any backend
 tree(Host, Name) ->
-    case gen_mod:db_type(serverhost(Host), ?MODULE) of
-	mnesia -> jlib:binary_to_atom(<<"nodetree_", Name/binary>>);
-	sql -> jlib:binary_to_atom(<<"nodetree_", Name/binary, "_sql">>);
-	_ -> Name
-    end.
+    submodule(Host, <<"nodetree_", Name/binary>>).
 
--spec plugin(host(), binary() | atom()) -> atom().
+-spec plugin(host(), binary()) -> atom().
 plugin(Host, Name) ->
-    case gen_mod:db_type(serverhost(Host), ?MODULE) of
-	mnesia -> jlib:binary_to_atom(<<"node_", Name/binary>>);
-	sql -> jlib:binary_to_atom(<<"node_", Name/binary, "_sql">>);
-	_ -> Name
-    end.
+    submodule(Host, <<"node_", Name/binary>>).
 
 -spec plugins(host()) -> [binary()].
 plugins(Host) ->
@@ -3505,14 +3432,16 @@ plugins(Host) ->
 	Plugins -> Plugins
     end.
 
--spec subscription_plugin(host()) -> pubsub_subscription |
-				     pubsub_subscription_sql |
-				     none.
+-spec subscription_plugin(host()) -> atom().
 subscription_plugin(Host) ->
+    submodule(Host, <<"pubsub_subscription">>).
+
+-spec submodule(host(), binary()) -> atom().
+submodule(Host, Name) ->
     case gen_mod:db_type(serverhost(Host), ?MODULE) of
-	mnesia -> pubsub_subscription;
-	sql -> pubsub_subscription_sql;
-	_ -> none
+	mnesia -> misc:binary_to_atom(Name);
+	Type -> misc:binary_to_atom(<<Name/binary, "_",
+		    (misc:atom_to_binary(Type))/binary>>)
     end.
 
 -spec config(binary(), any()) -> any().
@@ -3625,9 +3554,14 @@ tree_action(Host, Function, Args) ->
 		    {error, xmpp:err_internal_server_error(ErrTxt, ?MYLANG)}
 	    end;
 	Other ->
-	    ?ERROR_MSG("unsupported backend: ~p~n", [Other]),
-	    ErrTxt = <<"Database failure">>,
-	    {error, xmpp:err_internal_server_error(ErrTxt, ?MYLANG)}
+	    case catch Fun() of
+		{'EXIT', _} ->
+		    ?ERROR_MSG("unsupported backend: ~p~n", [Other]),
+		    ErrTxt = <<"Database failure">>,
+		    {error, xmpp:err_internal_server_error(ErrTxt, ?MYLANG)};
+		Result ->
+		    Result
+	    end
     end.
 
 %% @doc <p>node plugin call.</p>
@@ -3695,7 +3629,7 @@ transaction_retry(Host, ServerHost, Fun, Trans, DBType, Count) ->
 	    end,
 	    catch ejabberd_sql:SqlFun(ServerHost, Fun);
 	_ ->
-	    {unsupported, DBType}
+	    catch Fun()
     end,
     case Res of
 	{result, Result} ->

src/mod_register.erl

@@ -289,7 +289,7 @@ try_set_password(User, Server, Password, #iq{lang = Lang, meta = M} = IQ) ->
 		?INFO_MSG("~s has changed password from ~s",
 			  [jid:encode({User, Server, <<"">>}),
 			   ejabberd_config:may_hide_data(
-			     jlib:ip_to_list(maps:get(ip, M, {0,0,0,0})))]),
+			     misc:ip_to_list(maps:get(ip, M, {0,0,0,0})))]),
 		xmpp:make_iq_result(IQ);
 	    {error, empty_password} ->
 		Txt = <<"Empty password">>,
@@ -522,7 +522,7 @@ remove_timeout(Source) ->
     end.
 
 ip_to_string(Source) when is_tuple(Source) ->
-    jlib:ip_to_list(Source);
+    misc:ip_to_list(Source);
 ip_to_string(undefined) -> <<"undefined">>;
 ip_to_string(_) -> <<"unknown">>.
 
@@ -575,7 +575,7 @@ transform_ip_access(Opts) ->
                      "use access rules instead.", []),
         ACLs = lists:flatmap(
                  fun({Action, S}) ->
-                         ACLName = jlib:binary_to_atom(
+                         ACLName = misc:binary_to_atom(
                                      iolist_to_binary(
                                        ["ip_", S])),
                          [{Action, ACLName},

src/mod_s2s_dialback.erl

@@ -147,7 +147,7 @@ s2s_out_auth_result(#{db_enabled := true,
     %% Sending dialback request, section 2.1.1, step 1
     ?INFO_MSG("(~s) Retrying with s2s dialback authentication: ~s -> ~s (~s)",
 	      [SockMod:pp(Socket), LServer, RServer,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP))]),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP))]),
     State1 = maps:remove(stop_reason, State#{on_route => queue}),
     {stop, send_db_request(State1)};
 s2s_out_auth_result(State, _) ->
@@ -168,7 +168,7 @@ s2s_out_downgraded(#{db_enabled := true,
     ?INFO_MSG("(~s) Trying s2s dialback authentication with "
 	      "non-RFC compliant server: ~s -> ~s (~s)",
 	      [SockMod:pp(Socket), LServer, RServer,
-	       ejabberd_config:may_hide_data(jlib:ip_to_list(IP))]),
+	       ejabberd_config:may_hide_data(misc:ip_to_list(IP))]),
     {stop, send_db_request(State)};
 s2s_out_downgraded(State, _) ->
     State.

src/mod_shared_roster_ldap.erl

@@ -462,7 +462,7 @@ get_user_part_re(String, Pattern) ->
     end.
 
 parse_options(Host, Opts) ->
-    Eldap_ID = jlib:atom_to_binary(gen_mod:get_module_proc(Host, ?MODULE)),
+    Eldap_ID = misc:atom_to_binary(gen_mod:get_module_proc(Host, ?MODULE)),
     Cfg = eldap_utils:get_config(Host, Opts),
     GroupAttr = gen_mod:get_opt(ldap_groupattr, Opts,
                                 fun iolist_to_binary/1,

src/mod_shared_roster_sql.erl

@@ -67,7 +67,7 @@ groups_with_opts(Host) ->
     end.
 
 create_group(Host, Group, Opts) ->
-    SOpts = jlib:term_to_expr(Opts),
+    SOpts = misc:term_to_expr(Opts),
     F = fun () ->
 		?SQL_UPSERT_T(
                    "sr_group",
@@ -98,7 +98,7 @@ get_group_opts(Host, Group) ->
     end.
 
 set_group_opts(Host, Group, Opts) ->
-    SOpts = jlib:term_to_expr(Opts),
+    SOpts = misc:term_to_expr(Opts),
     F = fun () ->
 		?SQL_UPSERT_T(
                    "sr_group",
@@ -172,7 +172,7 @@ export(_Server) ->
     [{sr_group,
       fun(Host, #sr_group{group_host = {Group, LServer}, opts = Opts})
             when LServer == Host ->
-              SOpts = jlib:term_to_expr(Opts),
+              SOpts = misc:term_to_expr(Opts),
               [?SQL("delete from sr_group where name=%(Group)s;"),
                ?SQL("insert into sr_group(name, opts) values ("
                     "%(Group)s, %(SOpts)s);")];

src/mod_stream_mgmt.erl

@@ -36,6 +36,7 @@
 
 -include("xmpp.hrl").
 -include("logger.hrl").
+-include("p1_queue.hrl").
 
 -define(is_sm_packet(Pkt),
 	is_record(Pkt, sm_enable) or
@@ -44,7 +45,6 @@
 	is_record(Pkt, sm_r)).
 
 -type state() :: ejabberd_c2s:state().
--type lqueue() :: {non_neg_integer(), queue:queue()}.
 
 %%%===================================================================
 %%% API
@@ -102,6 +102,7 @@ c2s_stream_init({ok, State}, Opts) ->
 		    ({max_resume_timeout, _}) -> true;
 		    ({ack_timeout, _}) -> true;
 		    ({resend_on_timeout, _}) -> true;
+		    ({queue_type, _}) -> true;
 		    (_) -> false
 		 end, Opts),
     {ok, State#{mgmt_options => MgmtOpts}};
@@ -114,6 +115,7 @@ c2s_stream_started(#{lserver := LServer, mgmt_options := Opts} = State,
     ResumeTimeout = get_resume_timeout(LServer, Opts),
     MaxResumeTimeout = get_max_resume_timeout(LServer, Opts, ResumeTimeout),
     State1#{mgmt_state => inactive,
+	    mgmt_queue_type => get_queue_type(LServer, Opts),
 	    mgmt_max_queue => get_max_ack_queue(LServer, Opts),
 	    mgmt_timeout => ResumeTimeout,
 	    mgmt_max_timeout => MaxResumeTimeout,
@@ -216,9 +218,10 @@ c2s_handle_send(#{mgmt_state := MgmtState, mod := Mod,
 c2s_handle_send(State, _Pkt, _Result) ->
     State.
 
-c2s_handle_call(#{sid := {Time, _}, mod := Mod} = State,
+c2s_handle_call(#{sid := {Time, _}, mod := Mod, mgmt_queue := Queue} = State,
 		{resume_session, Time}, From) ->
-    Mod:reply(From, {resume, State}),
+    State1 = State#{mgmt_queue => p1_queue:file_to_ram(Queue)},
+    Mod:reply(From, {resume, State1}),
     {stop, State#{mgmt_state => resumed}};
 c2s_handle_call(#{mod := Mod} = State, {resume_session, _}, From) ->
     Mod:reply(From, {error, <<"Previous session not found">>}),
@@ -316,6 +319,7 @@ perform_stream_mgmt(Pkt, #{mgmt_xmlns := Xmlns} = State) ->
 
 -spec handle_enable(state(), sm_enable()) -> state().
 handle_enable(#{mgmt_timeout := DefaultTimeout,
+		mgmt_queue_type := QueueType,
 		mgmt_max_timeout := MaxTimeout,
 		mgmt_xmlns := Xmlns, jid := JID} = State,
 	      #sm_enable{resume = Resume, max = Max}) ->
@@ -339,7 +343,7 @@ handle_enable(#{mgmt_timeout := DefaultTimeout,
 		  #sm_enabled{xmlns = Xmlns}
 	  end,
     State1 = State#{mgmt_state => active,
-		    mgmt_queue => queue_new(),
+		    mgmt_queue => p1_queue:new(QueueType),
 		    mgmt_timeout => Timeout},
     send(State1, Res).
 
@@ -446,7 +450,7 @@ resend_rack(#{mgmt_ack_timer := _,
 	      mgmt_stanzas_out := NumStanzasOut,
 	      mgmt_stanzas_req := NumStanzasReq} = State) ->
     State1 = cancel_ack_timer(State),
-    case NumStanzasReq < NumStanzasOut andalso not queue_is_empty(Queue) of
+    case NumStanzasReq < NumStanzasOut andalso not p1_queue:is_empty(Queue) of
 	true -> send_rack(State1);
 	false -> State1
     end;
@@ -460,13 +464,13 @@ mgmt_queue_add(#{mgmt_stanzas_out := NumStanzasOut,
 		 4294967295 -> 0;
 		 Num -> Num + 1
 	     end,
-    Queue1 = queue_in({NewNum, p1_time_compat:timestamp(), Pkt}, Queue),
+    Queue1 = p1_queue:in({NewNum, p1_time_compat:timestamp(), Pkt}, Queue),
     State1 = State#{mgmt_queue => Queue1, mgmt_stanzas_out => NewNum},
     check_queue_length(State1).
 
 -spec mgmt_queue_drop(state(), non_neg_integer()) -> state().
 mgmt_queue_drop(#{mgmt_queue := Queue} = State, NumHandled) ->
-    NewQueue = queue_dropwhile(
+    NewQueue = p1_queue:dropwhile(
 		 fun({N, _T, _E}) -> N =< NumHandled end, Queue),
     State#{mgmt_queue => NewQueue}.
 
@@ -475,7 +479,7 @@ check_queue_length(#{mgmt_max_queue := Limit} = State)
   when Limit == infinity; Limit == exceeded ->
     State;
 check_queue_length(#{mgmt_queue := Queue, mgmt_max_queue := Limit} = State) ->
-    case queue_len(Queue) > Limit of
+    case p1_queue:len(Queue) > Limit of
 	true ->
 	    State#{mgmt_max_queue => exceeded};
 	false ->
@@ -484,14 +488,14 @@ check_queue_length(#{mgmt_queue := Queue, mgmt_max_queue := Limit} = State) ->
 
 -spec resend_unacked_stanzas(state()) -> state().
 resend_unacked_stanzas(#{mgmt_state := MgmtState,
-			 mgmt_queue := {QueueLen, _} = Queue,
+			 mgmt_queue := Queue,
 			 jid := JID} = State)
   when (MgmtState == active orelse
 	MgmtState == pending orelse
-	MgmtState == timeout) andalso QueueLen > 0 ->
+	MgmtState == timeout) andalso ?qlen(Queue) > 0 ->
     ?DEBUG("Resending ~B unacknowledged stanza(s) to ~s",
-	   [QueueLen, jid:encode(JID)]),
-    queue_foldl(
+	   [p1_queue:len(Queue), jid:encode(JID)]),
+    p1_queue:foldl(
       fun({_, Time, Pkt}, AccState) ->
 	      NewPkt = add_resent_delay_info(AccState, Pkt, Time),
 	      send(AccState, xmpp:put_meta(NewPkt, mgmt_is_resent, true))
@@ -504,11 +508,11 @@ route_unacked_stanzas(#{mgmt_state := MgmtState,
 			mgmt_resend := MgmtResend,
 			lang := Lang, user := User,
 			jid := JID, lserver := LServer,
-			mgmt_queue := {QueueLen, _} = Queue,
+			mgmt_queue := Queue,
 			resource := Resource} = State)
   when (MgmtState == active orelse
 	MgmtState == pending orelse
-	MgmtState == timeout) andalso QueueLen > 0 ->
+	MgmtState == timeout) andalso ?qlen(Queue) > 0 ->
     ResendOnTimeout = case MgmtResend of
 			  Resend when is_boolean(Resend) ->
 			      Resend;
@@ -522,8 +526,8 @@ route_unacked_stanzas(#{mgmt_state := MgmtState,
 			      end
 		      end,
     ?DEBUG("Re-routing ~B unacknowledged stanza(s) to ~s",
-	   [QueueLen, jid:encode(JID)]),
-    queue_foreach(
+	   [p1_queue:len(Queue), jid:encode(JID)]),
+    p1_queue:foreach(
       fun({_, _Time, #presence{from = From}}) ->
 	      ?DEBUG("Dropping presence stanza from ~s", [jid:encode(From)]);
 	 ({_, _Time, #iq{} = El}) ->
@@ -564,8 +568,9 @@ route_unacked_stanzas(_State) ->
 -spec inherit_session_state(state(), binary()) -> {ok, state()} |
 						  {error, binary()} |
 						  {error, binary(), non_neg_integer()}.
-inherit_session_state(#{user := U, server := S} = State, ResumeID) ->
-    case jlib:base64_to_term(ResumeID) of
+inherit_session_state(#{user := U, server := S,
+			mgmt_queue_type := QueueType} = State, ResumeID) ->
+    case misc:base64_to_term(ResumeID) of
 	{term, {R, Time}} ->
 	    case ejabberd_sm:get_session_pid(U, S, R) of
 		none ->
@@ -589,8 +594,12 @@ inherit_session_state(#{user := U, server := S} = State, ResumeID) ->
 				   mgmt_stanzas_in := NumStanzasIn,
 				   mgmt_stanzas_out := NumStanzasOut} = OldState} ->
 			    State1 = ejabberd_c2s:copy_state(State, OldState),
+			    Queue1 = case QueueType of
+					 ram -> Queue;
+					 _ -> p1_queue:ram_to_file(Queue)
+				     end,
 			    State2 = State1#{mgmt_xmlns => Xmlns,
-					     mgmt_queue => Queue,
+					     mgmt_queue => Queue1,
 					     mgmt_timeout => Timeout,
 					     mgmt_stanzas_in => NumStanzasIn,
 					     mgmt_stanzas_out => NumStanzasOut,
@@ -618,7 +627,7 @@ resume_session({Time, Pid}, _State) ->
 
 -spec make_resume_id(state()) -> binary().
 make_resume_id(#{sid := {Time, _}, resource := Resource}) ->
-    jlib:term_to_base64({Resource, Time}).
+    misc:term_to_base64({Resource, Time}).
 
 -spec add_resent_delay_info(state(), stanza(), erlang:timestamp()) -> stanza();
 			   (state(), xmlel(), erlang:timestamp()) -> xmlel().
@@ -632,44 +641,6 @@ add_resent_delay_info(_State, El, _Time) ->
 send(#{mod := Mod} = State, Pkt) ->
     Mod:send(State, Pkt).
 
--spec queue_new() -> lqueue().
-queue_new() ->
-    {0, queue:new()}.
-
--spec queue_in(term(), lqueue()) -> lqueue().
-queue_in(Elem, {N, Q}) ->
-    {N+1, queue:in(Elem, Q)}.
-
--spec queue_len(lqueue()) -> non_neg_integer().
-queue_len({N, _}) ->
-    N.
-
--spec queue_foldl(fun((term(), T) -> T), T, lqueue()) -> T.
-queue_foldl(F, Acc, {_N, Q}) ->
-    jlib:queue_foldl(F, Acc, Q).
-
--spec queue_foreach(fun((_) -> _), lqueue()) -> ok.
-queue_foreach(F, {_N, Q}) ->
-    jlib:queue_foreach(F, Q).
-
--spec queue_dropwhile(fun((term()) -> boolean()), lqueue()) -> lqueue().
-queue_dropwhile(F, {N, Q}) ->
-    case queue:peek(Q) of
-	{value, Item} ->
-	    case F(Item) of
-		true ->
-		    queue_dropwhile(F, {N-1, queue:drop(Q)});
-		false ->
-		    {N, Q}
-	    end;
-	empty ->
-	    {N, Q}
-    end.
-
--spec queue_is_empty(lqueue()) -> boolean().
-queue_is_empty({N, _Q}) ->
-    N == 0.
-
 -spec cancel_ack_timer(state()) -> state().
 cancel_ack_timer(#{mgmt_ack_timer := TRef} = State) ->
     case erlang:cancel_timer(TRef) of
@@ -741,6 +712,17 @@ get_resend_on_timeout(Host, Opts) ->
 	Resend -> Resend
     end.
 
+get_queue_type(Host, Opts) ->
+    VFun = mod_opt_type(queue_type),
+    case gen_mod:get_module_opt(Host, ?MODULE, queue_type, VFun) of
+	undefined ->
+	    case gen_mod:get_opt(queue_type, Opts, VFun) of
+		undefined -> ejabberd_config:default_queue_type(Host);
+		Type -> Type
+	    end;
+	Type -> Type
+    end.
+
 mod_opt_type(max_ack_queue) ->
     fun(I) when is_integer(I), I > 0 -> I;
        (infinity) -> infinity
@@ -757,6 +739,8 @@ mod_opt_type(resend_on_timeout) ->
     fun(B) when is_boolean(B) -> B;
        (if_offline) -> if_offline
     end;
+mod_opt_type(queue_type) ->
+    fun(ram) -> ram; (file) -> file end;
 mod_opt_type(_) ->
     [max_ack_queue, resume_timeout, max_resume_timeout, ack_timeout,
-     resend_on_timeout].
+     resend_on_timeout, queue_type].

src/mod_vcard_ldap.erl

@@ -362,7 +362,7 @@ parse_options(Host, Opts) ->
                               fun(infinity) -> 0;
                                  (I) when is_integer(I), I>0 -> I
                               end, 30),
-    Eldap_ID = jlib:atom_to_binary(gen_mod:get_module_proc(Host, ?PROCNAME)),
+    Eldap_ID = misc:atom_to_binary(gen_mod:get_module_proc(Host, ?PROCNAME)),
     Cfg = eldap_utils:get_config(Host, Opts),
     UIDsTemp = gen_mod:get_opt(
                  {ldap_uids, Host}, Opts,

src/mod_vcard_xupdate.erl

@@ -99,7 +99,7 @@ vcard_set(LUser, LServer, VCARD) ->
       <<>> -> remove_xupdate(LUser, LServer);
       BinVal ->
 	  add_xupdate(LUser, LServer,
-		      str:sha(jlib:decode_base64(BinVal)))
+		      str:sha(misc:decode_base64(BinVal)))
     end,
     ejabberd_sm:force_update_presence(US).
 

src/node_buddy.erl

@@ -39,6 +39,7 @@
     get_subscriptions/2, set_subscriptions/4,
     get_pending_nodes/2, get_states/1, get_state/2,
     set_state/1, get_items/7, get_items/3, get_item/7,
+    get_last_items/3,
     get_item/2, set_item/1, get_item_name/3, node_to_path/1,
     path_to_node/1]).
 
@@ -157,6 +158,9 @@ get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, SubId, RSM)
     node_flat:get_items(Nidx, JID, AccessModel,
 	PresenceSubscription, RosterGroup, SubId, RSM).
 
+get_last_items(Nidx, From, Count) ->
+    node_flat:get_last_items(Nidx, From, Count).
+
 get_item(Nidx, ItemId) ->
     node_flat:get_item(Nidx, ItemId).
 

src/node_club.erl

@@ -39,6 +39,7 @@
     get_subscriptions/2, set_subscriptions/4,
     get_pending_nodes/2, get_states/1, get_state/2,
     set_state/1, get_items/7, get_items/3, get_item/7,
+    get_last_items/3,
     get_item/2, set_item/1, get_item_name/3, node_to_path/1,
     path_to_node/1]).
 
@@ -156,6 +157,9 @@ get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, SubId, RSM)
     node_flat:get_items(Nidx, JID, AccessModel,
 	PresenceSubscription, RosterGroup, SubId, RSM).
 
+get_last_items(Nidx, From, Count) ->
+    node_flat:get_last_items(Nidx, From, Count).
+
 get_item(Nidx, ItemId) ->
     node_flat:get_item(Nidx, ItemId).
 

src/node_dag.erl

@@ -40,6 +40,7 @@
     get_subscriptions/2, set_subscriptions/4,
     get_pending_nodes/2, get_states/1, get_state/2,
     set_state/1, get_items/7, get_items/3, get_item/7,
+    get_last_items/3,
     get_item/2, set_item/1, get_item_name/3, node_to_path/1,
     path_to_node/1]).
 
@@ -144,6 +145,9 @@ get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, SubId, RSM)
     node_hometree:get_items(Nidx, JID, AccessModel,
 	PresenceSubscription, RosterGroup, SubId, RSM).
 
+get_last_items(Nidx, From, Count) ->
+    node_hometree:get_last_items(Nidx, From, Count).
+
 get_item(Nidx, ItemId) ->
     node_hometree:get_item(Nidx, ItemId).
 

src/node_dispatch.erl

@@ -46,6 +46,7 @@
     get_subscriptions/2, set_subscriptions/4,
     get_pending_nodes/2, get_states/1, get_state/2,
     set_state/1, get_items/7, get_items/3, get_item/7,
+    get_last_items/3,
     get_item/2, set_item/1, get_item_name/3, node_to_path/1,
     path_to_node/1]).
 
@@ -170,6 +171,9 @@ get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, SubId, RSM)
     node_hometree:get_items(Nidx, JID, AccessModel,
 	PresenceSubscription, RosterGroup, SubId, RSM).
 
+get_last_items(Nidx, From, Count) ->
+    node_hometree:get_last_items(Nidx, From, Count).
+
 get_item(Nidx, ItemId) ->
     node_hometree:get_item(Nidx, ItemId).
 

src/node_flat.erl

@@ -46,17 +46,18 @@
     get_subscriptions/2, set_subscriptions/4,
     get_pending_nodes/2, get_states/1, get_state/2,
     set_state/1, get_items/7, get_items/3, get_item/7,
+    get_last_items/3,
     get_item/2, set_item/1, get_item_name/3, node_to_path/1,
-    path_to_node/1, can_fetch_item/2, is_subscribed/1]).
+    path_to_node/1, can_fetch_item/2, is_subscribed/1, transform/1]).
 
 init(_Host, _ServerHost, _Opts) ->
     %pubsub_subscription:init(Host, ServerHost, Opts),
     ejabberd_mnesia:create(?MODULE, pubsub_state,
-	[{disc_copies, [node()]},
+	[{disc_copies, [node()]}, {index, [nodeidx]},
 	    {type, ordered_set},
 	    {attributes, record_info(fields, pubsub_state)}]),
     ejabberd_mnesia:create(?MODULE, pubsub_item,
-	[{disc_only_copies, [node()]},
+	[{disc_only_copies, [node()]}, {index, [nodeidx]},
 	    {attributes, record_info(fields, pubsub_item)}]),
     ejabberd_mnesia:create(?MODULE, pubsub_orphan,
 	[{disc_copies, [node()]},
@@ -131,7 +132,7 @@ create_node_permission(Host, ServerHost, _Node, _ParentNode, Owner, Access) ->
 create_node(Nidx, Owner) ->
     OwnerKey = jid:tolower(jid:remove_resource(Owner)),
     set_state(#pubsub_state{stateid = {OwnerKey, Nidx},
-	    affiliation = owner}),
+	    nodeidx = Nidx, affiliation = owner}),
     {result, {default, broadcast}}.
 
 delete_node(Nodes) ->
@@ -382,6 +383,7 @@ publish_item(Nidx, Publisher, PublishModel, MaxItems, ItemId, Payload,
 				payload = Payload};
 			_ ->
 			    #pubsub_item{itemid = {ItemId, Nidx},
+				nodeidx = Nidx,
 				creation = {Now, GenKey},
 				modification = PubId,
 				payload = Payload}
@@ -681,8 +683,7 @@ get_nodes_helper(NodeTree, #pubsub_state{stateid = {_, N}, subscriptions = Subs}
 %% ```get_states(Nidx) ->
 %%           node_default:get_states(Nidx).'''</p>
 get_states(Nidx) ->
-    States = case catch mnesia:match_object(
-	    #pubsub_state{stateid = {'_', Nidx}, _ = '_'}) of
+    States = case catch mnesia:index_read(pubsub_state, Nidx, #pubsub_state.nodeidx) of
 	List when is_list(List) -> List;
 	_ -> []
     end,
@@ -693,7 +694,7 @@ get_state(Nidx, Key) ->
     StateId = {Key, Nidx},
     case catch mnesia:read({pubsub_state, StateId}) of
 	[State] when is_record(State, pubsub_state) -> State;
-	_ -> #pubsub_state{stateid = StateId}
+	_ -> #pubsub_state{stateid = StateId, nodeidx = Nidx}
     end.
 
 %% @doc <p>Write a state into database.</p>
@@ -725,7 +726,7 @@ del_state(#pubsub_state{stateid = {Key, Nidx}, items = Items}) ->
 %% <p>PubSub plugins can store the items where they wants (for example in a
 %% relational database), or they can even decide not to persist any items.</p>
 get_items(Nidx, _From, _RSM) ->
-    Items = mnesia:match_object(#pubsub_item{itemid = {'_', Nidx}, _ = '_'}),
+    Items = mnesia:index_read(pubsub_item, Nidx, #pubsub_item.nodeidx),
     {result, {lists:reverse(lists:keysort(#pubsub_item.modification, Items)), undefined}}.
 
 get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, _SubId, RSM) ->
@@ -764,6 +765,12 @@ get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, _SubId, RSM
 	    get_items(Nidx, JID, RSM)
     end.
 
+get_last_items(Nidx, From, Count) when Count > 0 ->
+    {result, {Items, _}} = get_items(Nidx, From, undefined),
+    {result, lists:sublist(Items, Count)};
+get_last_items(_Nidx, _From, _Count) ->
+    {result, []}.
+
 %% @doc <p>Returns an item (one item list), given its reference.</p>
 
 get_item(Nidx, ItemId) ->
@@ -868,3 +875,10 @@ first_in_list(Pred, [H | T]) ->
 	true -> {value, H};
 	_ -> first_in_list(Pred, T)
     end.
+
+transform({pubsub_state, {Id, Nidx}, Is, A, Ss}) ->
+    {pubsub_state, {Id, Nidx}, Nidx, Is, A, Ss};
+transform({pubsub_item, {Id, Nidx}, C, M, P}) ->
+    {pubsub_item, {Id, Nidx}, Nidx, C, M, P};
+transform(Rec) ->
+    Rec.

src/node_flat_sql.erl

@@ -615,10 +615,11 @@ get_state_without_itemids(Nidx, JID) ->
 		"where nodeid=%(Nidx)d and jid=%(J)s")) of
 	{selected, [{SJID, Aff, Subs}]} ->
 	    #pubsub_state{stateid = {decode_jid(SJID), Nidx},
+			  nodeidx = Nidx,
 			  affiliation = decode_affiliation(Aff),
 			  subscriptions = decode_subscriptions(Subs)};
 	_ ->
-	    #pubsub_state{stateid = {JID, Nidx}}
+	    #pubsub_state{stateid = {JID, Nidx}, nodeidx = Nidx}
     end.
 
 set_state(State) ->
@@ -651,7 +652,7 @@ get_items(Nidx, From, undefined) ->
 		      ?SQL("select @(val)s from pubsub_node_option "
 			   "where nodeid=%(Nidx)d and name='max_items'")) of
 		   {selected, [{Value}]} ->
-		       jlib:expr_to_term(Value);
+		       misc:expr_to_term(Value);
 		   _ ->
 		       ?MAXITEMS
 	       end,
@@ -664,7 +665,7 @@ get_items(Nidx, _From, #rsm_set{max = Max, index = IncIndex,
 		      Before /= undefined -> {<<">">>, <<"asc">>};
 		      true -> {<<"is not">>, <<"desc">>}
 		   end,
-    SNidx = jlib:i2l(Nidx),
+    SNidx = misc:i2l(Nidx),
     I = if After /= undefined -> After;
 	   Before /= undefined -> Before;
 	   true -> undefined
@@ -773,8 +774,8 @@ get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, _SubId, RSM
     end.
 
 get_last_items(Nidx, _From, Count) ->
-    Limit = jlib:i2l(Count),
-    SNidx = jlib:i2l(Nidx),
+    Limit = misc:i2l(Count),
+    SNidx = misc:i2l(Nidx),
     Query = fun(mssql, _) ->
 		    ejabberd_sql:sql_query_t(
 		      [<<"select top ">>, Limit,
@@ -850,7 +851,7 @@ set_item(Item) ->
     Payload = Item#pubsub_item.payload,
     XML = str:join([fxml:element_to_binary(X) || X<-Payload], <<>>),
     S = fun ({T1, T2, T3}) ->
-	    str:join([jlib:i2l(T1, 6), jlib:i2l(T2, 6), jlib:i2l(T3, 6)], <<":">>)
+	    str:join([misc:i2l(T1, 6), misc:i2l(T2, 6), misc:i2l(T3, 6)], <<":">>)
     end,
     SM = S(M),
     SC = S(C),
@@ -876,7 +877,7 @@ del_items(Nidx, [ItemId]) ->
     del_item(Nidx, ItemId);
 del_items(Nidx, ItemIds) ->
     I = str:join([[<<"'">>, ejabberd_sql:escape(X), <<"'">>] || X <- ItemIds], <<",">>),
-    SNidx = jlib:i2l(Nidx),
+    SNidx = misc:i2l(Nidx),
     catch
     ejabberd_sql:sql_query_t([<<"delete from pubsub_item where itemid in (">>,
 	    I, <<") and nodeid='">>, SNidx, <<"';">>]).
@@ -1030,7 +1031,7 @@ raw_to_item(Nidx, {ItemId, SJID, Creation, Modification, XML}) ->
     JID = decode_jid(SJID),
     ToTime = fun (Str) ->
 	    [T1, T2, T3] = str:tokens(Str, <<":">>),
-	    {jlib:l2i(T1), jlib:l2i(T2), jlib:l2i(T3)}
+	    {misc:l2i(T1), misc:l2i(T2), misc:l2i(T3)}
     end,
     Payload = case fxml_stream:parse_element(XML) of
 	{error, _Reason} -> [];

src/node_hometree.erl

@@ -39,6 +39,7 @@
     get_subscriptions/2, set_subscriptions/4,
     get_pending_nodes/2, get_states/1, get_state/2,
     set_state/1, get_items/7, get_items/3, get_item/7,
+    get_last_items/3,
     get_item/2, set_item/1, get_item_name/3, node_to_path/1,
     path_to_node/1]).
 
@@ -154,6 +155,9 @@ get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, SubId, RSM)
     node_flat:get_items(Nidx, JID, AccessModel,
 	PresenceSubscription, RosterGroup, SubId, RSM).
 
+get_last_items(Nidx, From, Count) ->
+    node_flat:get_last_items(Nidx, From, Count).
+
 get_item(Nidx, ItemId) ->
     node_flat:get_item(Nidx, ItemId).
 

src/node_mb.erl

@@ -53,6 +53,7 @@
     get_subscriptions/2, set_subscriptions/4,
     get_pending_nodes/2, get_states/1, get_state/2,
     set_state/1, get_items/7, get_items/3, get_item/7,
+    get_last_items/3,
     get_item/2, set_item/1, get_item_name/3, node_to_path/1,
     path_to_node/1]).
 
@@ -171,6 +172,9 @@ get_items(Nidx, From, RSM) ->
 get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, SubId, RSM) ->
     node_pep:get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, SubId, RSM).
 
+get_last_items(Nidx, From, Count) ->
+    node_pep:get_last_items(Nidx, From, Count).
+
 get_item(Nidx, ItemId) ->
     node_pep:get_item(Nidx, ItemId).
 

src/node_mix.erl

@@ -37,6 +37,7 @@
     get_subscriptions/2, set_subscriptions/4,
     get_pending_nodes/2, get_states/1, get_state/2,
     set_state/1, get_items/7, get_items/3, get_item/7,
+    get_last_items/3,
     get_item/2, set_item/1, get_item_name/3, node_to_path/1,
     path_to_node/1]).
 
@@ -161,6 +162,9 @@ get_items(Nidx, JID, AccessModel, PresenceSubscription, RosterGroup, SubId, RSM)
     node_flat:get_items(Nidx, JID, AccessModel,
 	PresenceSubscription, RosterGroup, SubId, RSM).
 
+get_last_items(Nidx, From, Count) ->
+    node_flat:get_last_items(Nidx, From, Count).
+
 get_item(Nidx, ItemId) ->
     node_flat:get_item(Nidx, ItemId).