diff --git a/CHANGES-5.0.md b/CHANGES-5.0.md index 32ab74076..fcf2154d5 100644 --- a/CHANGES-5.0.md +++ b/CHANGES-5.0.md @@ -19,6 +19,7 @@ * Improve the dashboard listener startup log, the listener name is no longer spliced with port information, and the colon(:) is no longer displayed when IP is not specified. [#8480](https://github.com/emqx/emqx/pull/8480) * Remove `/configs/listeners` API, use `/listeners/` instead. [#8485](https://github.com/emqx/emqx/pull/8485) +* Optimize performance of builtin database operations in processes with long message queue [8439](https://github.com/emqx/emqx/pull/8439) # 5.0.3 @@ -35,13 +36,13 @@ # 5.0.2 -Announcemnet: EMQX team has decided to stop supporting relup for opensouce edition. -Going forward, it will be an enterprise only feature. +Announcement: EMQX team has decided to stop supporting relup for opensource edition. +Going forward, it will be an enterprise-only feature. Main reason: relup requires carefully crafted upgrade instructions from ALL previous versions. For example, 4.3 is now at 4.3.16, we have `4.3.0->4.3.16`, `4.3.1->4.3.16`, ... 16 such upgrade paths in total to maintain. -This had been the biggest obstacle for EMQX team to act agile enough in deliverying enhancements and fixes. +This had been the biggest obstacle for EMQX team to act agile enough in delivering enhancements and fixes. ## Enhancements diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 888ddb15d..0c0a09cc0 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -27,7 +27,7 @@ {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}, {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}, {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}, - {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.1"}}}, + {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.3"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 2478a2540..6353a4efa 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -262,6 +262,8 @@ fast_forward_to_commit(Node, ToTnxId) -> %% @private init([Node, RetryMs]) -> + %% Workaround for https://github.com/emqx/mria/issues/94: + _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs}, diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index a4946e8cf..1441a4180 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib]}, diff --git a/mix.exs b/mix.exs index b681c0960..f4ed355ed 100644 --- a/mix.exs +++ b/mix.exs @@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true}, - {:ekka, github: "emqx/ekka", tag: "0.13.1", override: true}, + {:ekka, github: "emqx/ekka", tag: "0.13.2", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:minirest, github: "emqx/minirest", tag: "1.3.5", override: true}, {:ecpool, github: "emqx/ecpool", tag: "0.5.2"}, @@ -170,35 +170,40 @@ defmodule EMQXUmbrella.MixProject do hocon: :load, emqx: :load, emqx_conf: :load, - emqx_machine: :permanent, - mria: :load, - mnesia: :load, - ekka: :load, - emqx_plugin_libs: :load, - esasl: :load, - observer_cli: :permanent, - system_monitor: :load, - emqx_http_lib: :permanent, - emqx_resource: :permanent, - emqx_connector: :permanent, - emqx_authn: :permanent, - emqx_authz: :permanent, - emqx_auto_subscribe: :permanent, - emqx_gateway: :permanent, - emqx_exhook: :permanent, - emqx_bridge: :permanent, - emqx_rule_engine: :permanent, - emqx_modules: :permanent, - emqx_management: :permanent, - emqx_dashboard: :permanent, - emqx_retainer: :permanent, - emqx_statsd: :permanent, - emqx_prometheus: :permanent, - emqx_psk: :permanent, - emqx_slow_subs: :permanent, - emqx_plugins: :permanent, - emqx_mix: :none + emqx_machine: :permanent ] ++ + if(enable_rocksdb?(), + do: [mnesia_rocksdb: :load], + else: [] + ) ++ + [ + mnesia: :load, + ekka: :load, + emqx_plugin_libs: :load, + esasl: :load, + observer_cli: :permanent, + system_monitor: :load, + emqx_http_lib: :permanent, + emqx_resource: :permanent, + emqx_connector: :permanent, + emqx_authn: :permanent, + emqx_authz: :permanent, + emqx_auto_subscribe: :permanent, + emqx_gateway: :permanent, + emqx_exhook: :permanent, + emqx_bridge: :permanent, + emqx_rule_engine: :permanent, + emqx_modules: :permanent, + emqx_management: :permanent, + emqx_dashboard: :permanent, + emqx_retainer: :permanent, + emqx_statsd: :permanent, + emqx_prometheus: :permanent, + emqx_psk: :permanent, + emqx_slow_subs: :permanent, + emqx_plugins: :permanent, + emqx_mix: :none + ] ++ if(enable_quicer?(), do: [quicer: :permanent], else: []) ++ if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++ if(enable_jq?(), do: [jq: :permanent], else: []) ++ @@ -616,6 +621,11 @@ defmodule EMQXUmbrella.MixProject do ]) or "1" == System.get_env("BUILD_WITH_QUIC") end + defp enable_rocksdb?() do + not build_without_rocksdb?() or + "1" == System.get_env("BUILD_WITH_QUIC") + end + defp pkg_vsn() do %{edition_type: edition_type} = check_profile!() basedir = Path.dirname(__ENV__.file) @@ -657,6 +667,12 @@ defmodule EMQXUmbrella.MixProject do String.downcase(opt) != "false" end + defp build_without_rocksdb?() do + opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false") + + String.downcase(opt) != "false" + end + defp from_rebar_to_eex_template(str) do # we must not consider surrounding space in the template var name # because some help strings contain informative variables that diff --git a/rebar.config b/rebar.config index d7d168f7f..7d2570636 100644 --- a/rebar.config +++ b/rebar.config @@ -54,7 +54,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}} - , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.1"}}} + , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.2"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.5"}}} , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}} diff --git a/rebar.config.erl b/rebar.config.erl index d88574a14..810c16207 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -88,6 +88,10 @@ is_quicer_supported() -> is_win32() orelse is_centos_6()) orelse "1" == os:getenv("BUILD_WITH_QUIC"). +is_rocksdb_supported() -> + not (false =/= os:getenv("BUILD_WITHOUT_ROCKSDB")) orelse + "1" == os:getenv("BUILD_WITH_ROCKSDB"). + is_macos() -> {unix, darwin} =:= os:type(). @@ -318,34 +322,37 @@ relx_apps(ReleaseType, Edition) -> % started by emqx_machine {emqx, load}, {emqx_conf, load}, - emqx_machine, - {mnesia, load}, - {ekka, load}, - {emqx_plugin_libs, load}, - {esasl, load}, - observer_cli, - % started by emqx_machine - {system_monitor, load}, - emqx_http_lib, - emqx_resource, - emqx_connector, - emqx_authn, - emqx_authz, - emqx_auto_subscribe, - emqx_gateway, - emqx_exhook, - emqx_bridge, - emqx_rule_engine, - emqx_modules, - emqx_management, - emqx_dashboard, - emqx_retainer, - emqx_statsd, - emqx_prometheus, - emqx_psk, - emqx_slow_subs, - emqx_plugins + emqx_machine ] ++ + [{mnesia_rocksdb, load} || is_rocksdb_supported()] ++ + [ + {mnesia, load}, + {ekka, load}, + {emqx_plugin_libs, load}, + {esasl, load}, + observer_cli, + % started by emqx_machine + {system_monitor, load}, + emqx_http_lib, + emqx_resource, + emqx_connector, + emqx_authn, + emqx_authz, + emqx_auto_subscribe, + emqx_gateway, + emqx_exhook, + emqx_bridge, + emqx_rule_engine, + emqx_modules, + emqx_management, + emqx_dashboard, + emqx_retainer, + emqx_statsd, + emqx_prometheus, + emqx_psk, + emqx_slow_subs, + emqx_plugins + ] ++ [quicer || is_quicer_supported()] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ [jq || is_jq_supported()] ++