From 7966aaad5f6e7268c143215639b543beffce8256 Mon Sep 17 00:00:00 2001 From: Kinplemelon Date: Thu, 21 Dec 2023 18:14:26 +0800 Subject: [PATCH 1/8] chore: upgrade dashboard to e1.4.0 for ee and v1.6.0 for ce --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index beab450a1..6897d7f5a 100644 --- a/Makefile +++ b/Makefile @@ -20,8 +20,8 @@ endif # Dashboard version # from https://github.com/emqx/emqx-dashboard5 -export EMQX_DASHBOARD_VERSION ?= v1.5.2 -export EMQX_EE_DASHBOARD_VERSION ?= e1.4.0-beta.8 +export EMQX_DASHBOARD_VERSION ?= v1.6.0 +export EMQX_EE_DASHBOARD_VERSION ?= e1.4.0 PROFILE ?= emqx REL_PROFILES := emqx emqx-enterprise From 5d1a412d0c2e278db861378481b8f5daaf470e1a Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 21 Dec 2023 16:37:18 +0800 Subject: [PATCH 2/8] fix: redis resource_opts not working --- .../src/emqx_bridge_redis_action_info.erl | 36 +++++++++---------- .../src/emqx_mgmt_api_configs.erl | 7 ++-- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl index 690faafac..6a268c931 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl @@ -11,8 +11,8 @@ action_type_name/0, connector_type_name/0, schema_module/0, - bridge_v1_config_to_action_config/2, connector_action_config_to_bridge_v1_config/2, + bridge_v1_config_to_action_config/2, bridge_v1_config_to_connector_config/1, bridge_v1_type_name_fun/1 ]). @@ -28,14 +28,25 @@ connector_type_name() -> redis. schema_module() -> ?SCHEMA_MODULE. +%% redis_cluster don't have batch options connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) -> - maps:merge( + Config0 = emqx_utils_maps:deep_merge( maps:without( [<<"connector">>], - map_unindent(<<"parameters">>, ActionConfig) + emqx_utils_maps:unindent(<<"parameters">>, ActionConfig) ), - map_unindent(<<"parameters">>, ConnectorConfig) - ). + emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig) + ), + Config1 = + case Config0 of + #{<<"resource_opts">> := ResOpts0, <<"redis_type">> := Type} -> + Schema = emqx_bridge_redis:fields("creation_opts_redis_" ++ binary_to_list(Type)), + ResOpts = maps:with(schema_keys(Schema), ResOpts0), + Config0#{<<"resource_opts">> => ResOpts}; + _ -> + Config0 + end, + maps:without([<<"description">>], Config1). bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) -> ActionTopLevelKeys = schema_keys(?SCHEMA_MODULE:fields(redis_action)), @@ -81,22 +92,9 @@ v1_type(<<"cluster">>) -> redis_cluster. bridge_v1_type_names() -> [redis_single, redis_sentinel, redis_cluster]. -map_unindent(Key, Map) -> - maps:merge( - maps:get(Key, Map), - maps:remove(Key, Map) - ). - -map_indent(IndentKey, PickKeys, Map) -> - maps:put( - IndentKey, - maps:with(PickKeys, Map), - maps:without(PickKeys, Map) - ). - schema_keys(Schema) -> [bin(Key) || {Key, _} <- Schema]. make_config_map(PickKeys, IndentKeys, Config) -> Conf0 = maps:with(PickKeys, Config), - map_indent(<<"parameters">>, IndentKeys, Conf0). + emqx_utils_maps:indent(<<"parameters">>, IndentKeys, Conf0). diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index e07895ef0..22fcdd907 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -356,10 +356,13 @@ configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> case emqx_conf_cli:load_config(Conf, #{mode => Mode, log => none}) of ok -> {200}; - {error, MsgList} -> + %% bad hocon format + {error, MsgList = [{_, _} | _]} -> JsonFun = fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end, JsonMap = emqx_utils_maps:jsonable_map(maps:from_list(MsgList), JsonFun), - {400, #{<<"content-type">> => <<"text/plain">>}, JsonMap} + {400, #{<<"content-type">> => <<"text/plain">>}, JsonMap}; + {error, Msg} -> + {400, #{<<"content-type">> => <<"text/plain">>}, Msg} end. find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Preferences) > 0 -> From 37930f1d3c0ef538a88dc3c156712a06f5e0bbaa Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 21 Dec 2023 19:46:22 +0800 Subject: [PATCH 3/8] fix: emqx conf ctl load failed with connectors/actions --- apps/emqx_bridge/src/emqx_bridge_v2.erl | 102 ++++++++++-------- apps/emqx_conf/src/emqx_conf_cli.erl | 17 ++- apps/emqx_connector/src/emqx_connector.erl | 33 +++--- .../test/emqx_mgmt_api_configs_SUITE.erl | 9 +- 4 files changed, 96 insertions(+), 65 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge_v2.erl b/apps/emqx_bridge/src/emqx_bridge_v2.erl index 9436ac0ea..cd6172eda 100644 --- a/apps/emqx_bridge/src/emqx_bridge_v2.erl +++ b/apps/emqx_bridge/src/emqx_bridge_v2.erl @@ -416,7 +416,7 @@ uninstall_bridge_v2( {error, _} -> ok; ok -> - %% Deinstall from connector + %% uninstall from connector ConnectorId = emqx_connector_resource:resource_id( connector_type(BridgeV2Type), ConnectorName ), @@ -869,6 +869,8 @@ config_key_path() -> config_key_path_leaf() -> [?ROOT_KEY, '?', '?']. +pre_config_update(_, {force_update, Conf}, _OldConf) -> + {ok, Conf}; %% NOTE: We depend on the `emqx_bridge:pre_config_update/3` to restart/stop the %% underlying resources. pre_config_update(_, {_Oper, _, _}, undefined) -> @@ -882,55 +884,15 @@ pre_config_update(_Path, Conf, _OldConfig) when is_map(Conf) -> operation_to_enable(disable) -> false; operation_to_enable(enable) -> true. +%% A public API that can trigger this is: +%% bin/emqx ctl conf load data/configs/cluster.hocon +post_config_update([?ROOT_KEY], {force_update, _Req}, NewConf, OldConf, _AppEnv) -> + do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors => false}); %% This top level handler will be triggered when the actions path is updated %% with calls to emqx_conf:update([actions], BridgesConf, #{}). %% -%% A public API that can trigger this is: -%% bin/emqx ctl conf load data/configs/cluster.hocon post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> - #{added := Added, removed := Removed, changed := Updated} = - diff_confs(NewConf, OldConf), - %% new and updated bridges must have their connector references validated - UpdatedConfigs = - lists:map( - fun({{Type, BridgeName}, {_Old, New}}) -> - {Type, BridgeName, New} - end, - maps:to_list(Updated) - ), - AddedConfigs = - lists:map( - fun({{Type, BridgeName}, AddedConf}) -> - {Type, BridgeName, AddedConf} - end, - maps:to_list(Added) - ), - ToValidate = UpdatedConfigs ++ AddedConfigs, - case multi_validate_referenced_connectors(ToValidate) of - ok -> - %% The config update will be failed if any task in `perform_bridge_changes` failed. - RemoveFun = fun uninstall_bridge_v2/3, - CreateFun = fun install_bridge_v2/3, - UpdateFun = fun(Type, Name, {OldBridgeConf, Conf}) -> - uninstall_bridge_v2(Type, Name, OldBridgeConf), - install_bridge_v2(Type, Name, Conf) - end, - Result = perform_bridge_changes([ - #{action => RemoveFun, data => Removed}, - #{ - action => CreateFun, - data => Added, - on_exception_fn => fun emqx_bridge_resource:remove/4 - }, - #{action => UpdateFun, data => Updated} - ]), - ok = unload_message_publish_hook(), - ok = load_message_publish_hook(NewConf), - ?tp(bridge_post_config_update_done, #{}), - Result; - {error, Error} -> - {error, Error} - end; + do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors => true}); post_config_update([?ROOT_KEY, BridgeType, BridgeName], '$remove', _, _OldConf, _AppEnvs) -> Conf = emqx:get_config([?ROOT_KEY, BridgeType, BridgeName]), ok = uninstall_bridge_v2(BridgeType, BridgeName, Conf), @@ -970,6 +932,50 @@ post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, OldConf, {error, Error} end. +do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors := NeedValidate}) -> + #{added := Added, removed := Removed, changed := Updated} = + diff_confs(NewConf, OldConf), + UpdatedConfigs = + lists:map( + fun({{Type, BridgeName}, {_Old, New}}) -> + {Type, BridgeName, New} + end, + maps:to_list(Updated) + ), + AddedConfigs = + lists:map( + fun({{Type, BridgeName}, AddedConf}) -> + {Type, BridgeName, AddedConf} + end, + maps:to_list(Added) + ), + ToValidate = UpdatedConfigs ++ AddedConfigs, + case multi_validate_referenced_connectors(NeedValidate, ToValidate) of + ok -> + %% The config update will be failed if any task in `perform_bridge_changes` failed. + RemoveFun = fun uninstall_bridge_v2/3, + CreateFun = fun install_bridge_v2/3, + UpdateFun = fun(Type, Name, {OldBridgeConf, Conf}) -> + uninstall_bridge_v2(Type, Name, OldBridgeConf), + install_bridge_v2(Type, Name, Conf) + end, + Result = perform_bridge_changes([ + #{action => RemoveFun, data => Removed}, + #{ + action => CreateFun, + data => Added, + on_exception_fn => fun emqx_bridge_resource:remove/4 + }, + #{action => UpdateFun, data => Updated} + ]), + ok = unload_message_publish_hook(), + ok = load_message_publish_hook(NewConf), + ?tp(bridge_post_config_update_done, #{}), + Result; + {error, Error} -> + {error, Error} + end. + diff_confs(NewConfs, OldConfs) -> emqx_utils_maps:diff_maps( flatten_confs(NewConfs), @@ -1600,7 +1606,9 @@ to_connector(ConnectorNameBin, BridgeType) -> throw(not_found) end. -multi_validate_referenced_connectors(Configs) -> +multi_validate_referenced_connectors(false, _Configs) -> + ok; +multi_validate_referenced_connectors(true, Configs) -> Pipeline = lists:map( fun({Type, BridgeName, #{connector := ConnectorName}}) -> diff --git a/apps/emqx_conf/src/emqx_conf_cli.erl b/apps/emqx_conf/src/emqx_conf_cli.erl index 07f5b034d..f2ced3327 100644 --- a/apps/emqx_conf/src/emqx_conf_cli.erl +++ b/apps/emqx_conf/src/emqx_conf_cli.erl @@ -286,9 +286,16 @@ update_config_cluster( check_res(Key, emqx_authn:merge_config(Conf), Conf, Opts); update_config_cluster(Key, NewConf, #{mode := merge} = Opts) -> Merged = merge_conf(Key, NewConf), - check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS), NewConf, Opts); + Request = make_request(Key, Merged), + check_res(Key, emqx_conf:update([Key], Request, ?OPTIONS), NewConf, Opts); update_config_cluster(Key, Value, #{mode := replace} = Opts) -> - check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS), Value, Opts). + Request = make_request(Key, Value), + check_res(Key, emqx_conf:update([Key], Request, ?OPTIONS), Value, Opts). + +make_request(Key, Value) when Key =:= <<"connectors">> orelse Key =:= <<"actions">> -> + {force_update, Value}; +make_request(_Key, Value) -> + Value. -define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}). update_config_local( @@ -305,9 +312,11 @@ update_config_local( check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS), Conf, Opts); update_config_local(Key, NewConf, #{mode := merge} = Opts) -> Merged = merge_conf(Key, NewConf), - check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS), NewConf, Opts); + Request = make_request(Key, Merged), + check_res(node(), Key, emqx:update_config([Key], Request, ?LOCAL_OPTIONS), NewConf, Opts); update_config_local(Key, Value, #{mode := replace} = Opts) -> - check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS), Value, Opts). + Request = make_request(Key, Value), + check_res(node(), Key, emqx:update_config([Key], Request, ?LOCAL_OPTIONS), Value, Opts). check_res(Key, Res, Conf, Opts) -> check_res(cluster, Key, Res, Conf, Opts). check_res(Node, Key, {ok, _}, _Conf, Opts) -> diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl index 30654bb13..dac85273a 100644 --- a/apps/emqx_connector/src/emqx_connector.erl +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -107,6 +107,8 @@ config_key_path() -> pre_config_update([?ROOT_KEY], RawConf, RawConf) -> {ok, RawConf}; +pre_config_update([?ROOT_KEY], {force_update, NewConf}, RawConf) -> + pre_config_update([?ROOT_KEY], NewConf, RawConf); pre_config_update([?ROOT_KEY], NewConf, _RawConf) -> case multi_validate_connector_names(NewConf) of ok -> @@ -135,23 +137,16 @@ pre_config_update(Path, Conf, _OldConfig) when is_map(Conf) -> operation_to_enable(disable) -> false; operation_to_enable(enable) -> true. +post_config_update([?ROOT_KEY], {force_update, _}, NewConf, OldConf, _AppEnv) -> + #{added := Added, removed := Removed, changed := Updated} = + diff_confs(NewConf, OldConf), + perform_connector_changes(Removed, Added, Updated); post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) -> #{added := Added, removed := Removed, changed := Updated} = diff_confs(NewConf, OldConf), case ensure_no_channels(Removed) of ok -> - %% The config update will be failed if any task in `perform_connector_changes` failed. - Result = perform_connector_changes([ - #{action => fun emqx_connector_resource:remove/4, data => Removed}, - #{ - action => fun emqx_connector_resource:create/4, - data => Added, - on_exception_fn => fun emqx_connector_resource:remove/4 - }, - #{action => fun emqx_connector_resource:update/4, data => Updated} - ]), - ?tp(connector_post_config_update_done, #{}), - Result; + perform_connector_changes(Removed, Added, Updated); {error, Error} -> {error, Error} end; @@ -175,6 +170,20 @@ post_config_update([?ROOT_KEY, Type, Name], _Req, NewConf, OldConf, _AppEnvs) -> ?tp(connector_post_config_update_done, #{}), ok. +%% The config update will be failed if any task in `perform_connector_changes` failed. +perform_connector_changes(Removed, Added, Updated) -> + Result = perform_connector_changes([ + #{action => fun emqx_connector_resource:remove/4, data => Removed}, + #{ + action => fun emqx_connector_resource:create/4, + data => Added, + on_exception_fn => fun emqx_connector_resource:remove/4 + }, + #{action => fun emqx_connector_resource:update/4, data => Updated} + ]), + ?tp(connector_post_config_update_done, #{}), + Result. + list() -> maps:fold( fun(Type, NameAndConf, Connectors) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index ed50ecba4..2bfff17a6 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -378,8 +378,13 @@ t_get_configs_in_different_accept(_Config) -> ?assertMatch({400, "application/json", _}, Request(<<"application/xml">>)). t_create_webhook_v1_bridges_api({'init', Config}) -> - application:ensure_all_started(emqx_connector), - application:ensure_all_started(emqx_bridge), + lists:foreach( + fun(App) -> + _ = application:stop(App), + {ok, [App]} = application:ensure_all_started(App) + end, + [emqx_connector, emqx_bridge] + ), Config; t_create_webhook_v1_bridges_api({'end', _}) -> application:stop(emqx_bridge), From 85d7518a7d4065ee53b8b18ae08934c91d0b35bb Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Wed, 20 Dec 2023 22:33:12 +0100 Subject: [PATCH 4/8] feat(schema): provide type-level documentation snippets For stuff like `duration()`, `bytesize()` and `secret()` for now. --- apps/emqx_conf/src/emqx_conf.erl | 41 +++++++++++++------ apps/emqx_conf/src/emqx_conf_schema_types.erl | 24 +++++++---- rel/i18n/emqx_conf_schema_types.hocon | 12 ++++++ scripts/schema-dump-reformat.escript | 3 +- 4 files changed, 58 insertions(+), 22 deletions(-) create mode 100644 rel/i18n/emqx_conf_schema_types.hocon diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 140b008d1..4f845a166 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -28,7 +28,7 @@ -export([remove/2, remove/3]). -export([tombstone/2]). -export([reset/2, reset/3]). --export([dump_schema/2, reformat_schema_dump/1]). +-export([dump_schema/2, reformat_schema_dump/2]). -export([schema_module/0]). %% TODO: move to emqx_dashboard when we stop building api schema at build time @@ -186,7 +186,7 @@ gen_schema_json(Dir, SchemaModule, Lang) -> ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang). gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) -> - NestedStruct = reformat_schema_dump(StructsJsonArray), + NestedStruct = reformat_schema_dump(StructsJsonArray, Lang), %% write to files NestedJsonFile = filename:join([Dir, "schema-v2-" ++ Lang ++ ".json"]), io:format(user, "===< Generating: ~s~n", [NestedJsonFile]), @@ -196,15 +196,17 @@ gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) -> ok. %% @doc This function is exported for scripts/schema-dump-reformat.escript -reformat_schema_dump(StructsJsonArray0) -> +reformat_schema_dump(StructsJsonArray0, Lang) -> %% prepare + DescResolver = make_desc_resolver(Lang), StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0), #{fields := RootFields} = hd(StructsJsonArray), RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields), RootNames = lists:map(fun to_bin/1, RootNames0), %% reformat [Root | FlatStructs0] = lists:map( - fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray + fun(Struct) -> gen_flat_doc(RootNames, Struct, DescResolver) end, + StructsJsonArray ), FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0], gen_nested_doc(FlatStructs). @@ -302,7 +304,7 @@ expand_ref(#{hash := FullName}, FindFn, Path) -> %% generate flat docs for each struct. %% using references to link to other structs. -gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) -> +gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S, DescResolver) -> ShortName = short_name(FullName), case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of true -> @@ -314,18 +316,17 @@ gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) -> text => short_name(FullName), hash => format_hash(FullName), doc => maps:get(desc, S, <<"">>), - fields => format_fields(Fields) + fields => format_fields(Fields, DescResolver) }. -format_fields([]) -> - []; -format_fields([Field | Fields]) -> - [format_field(Field) | format_fields(Fields)]. +format_fields(Fields, DescResolver) -> + [format_field(F, DescResolver) || F <- Fields]. -format_field(#{name := Name, aliases := Aliases, type := Type} = F) -> +format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver) -> L = [ {text, Name}, {type, format_type(Type)}, + {typedoc, format_type_desc(Type, DescResolver)}, {refs, format_refs(Type)}, {aliases, case Aliases of @@ -393,10 +394,26 @@ format_union_members([Member | Members], Acc) -> NewAcc = [format_type(Member) | Acc], format_union_members(Members, NewAcc). +format_type_desc(#{kind := primitive, name := Name}, DescResolver) -> + format_primitive_type_desc(Name, DescResolver); +format_type_desc(#{}, _DescResolver) -> + undefined. + format_primitive_type(TypeStr) -> - Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr), + Spec = get_primitive_typespec(TypeStr), to_bin(maps:get(type, Spec)). +format_primitive_type_desc(TypeStr, DescResolver) -> + case get_primitive_typespec(TypeStr) of + #{desc := Desc} -> + DescResolver(Desc); + #{} -> + undefined + end. + +get_primitive_typespec(TypeStr) -> + emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr). + %% All types should have a namespace to avlid name clashing. is_missing_namespace(ShortName, FullName, RootNames) -> case lists:member(ShortName, RootNames) of diff --git a/apps/emqx_conf/src/emqx_conf_schema_types.erl b/apps/emqx_conf/src/emqx_conf_schema_types.erl index 239624a81..9d00b9650 100644 --- a/apps/emqx_conf/src/emqx_conf_schema_types.erl +++ b/apps/emqx_conf/src/emqx_conf_schema_types.erl @@ -16,6 +16,8 @@ -module(emqx_conf_schema_types). +-include_lib("hocon/include/hocon_types.hrl"). + -export([readable/2]). -export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]). @@ -165,37 +167,37 @@ readable("duration()") -> #{ swagger => #{type => string, example => <<"12m">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"12m">>} + docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)} }; readable("duration_s()") -> #{ swagger => #{type => string, example => <<"1h">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"1h">>} + docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)} }; readable("duration_ms()") -> #{ swagger => #{type => string, example => <<"32s">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"32s">>} + docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)} }; readable("timeout_duration()") -> #{ swagger => #{type => string, example => <<"12m">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"12m">>} + docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)} }; readable("timeout_duration_s()") -> #{ swagger => #{type => string, example => <<"1h">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"1h">>} + docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)} }; readable("timeout_duration_ms()") -> #{ swagger => #{type => string, example => <<"32s">>}, dashboard => #{type => duration}, - docgen => #{type => "String", example => <<"32s">>} + docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)} }; readable("percent()") -> #{ @@ -219,13 +221,13 @@ readable("bytesize()") -> #{ swagger => #{type => string, example => <<"32MB">>}, dashboard => #{type => 'byteSize'}, - docgen => #{type => "String", example => <<"32MB">>} + docgen => #{type => "Bytesize", example => <<"32MB">>, desc => ?DESC(bytesize)} }; readable("wordsize()") -> #{ swagger => #{type => string, example => <<"1024KB">>}, dashboard => #{type => 'wordSize'}, - docgen => #{type => "String", example => <<"1024KB">>} + docgen => #{type => "Bytesize", example => <<"1024KB">>, desc => ?DESC(bytesize)} }; readable("map(" ++ Map) -> [$) | _MapArgs] = lists:reverse(Map), @@ -287,7 +289,11 @@ readable("secret()") -> #{ swagger => #{type => string, example => <<"R4ND0M/S∃CЯ∃T"/utf8>>}, dashboard => #{type => string}, - docgen => #{type => "String", example => <<"R4ND0M/S∃CЯ∃T"/utf8>>} + docgen => #{ + type => "Secret", + example => <<"R4ND0M/S∃CЯ∃T"/utf8>>, + desc => ?DESC(secret) + } }; readable(TypeStr0) -> case string:split(TypeStr0, ":") of diff --git a/rel/i18n/emqx_conf_schema_types.hocon b/rel/i18n/emqx_conf_schema_types.hocon new file mode 100644 index 000000000..4955d3173 --- /dev/null +++ b/rel/i18n/emqx_conf_schema_types.hocon @@ -0,0 +1,12 @@ +emqx_conf_schema_types { + + duration.desc: + """A string that represents a time duration, for example: 10s, 2.5m, 1h30m, 1W2D, or 2345ms, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing 1200ms for Duration(s) is equivalent to writing 1s. It doesn't matter if units are in upper or lower case.""" + + bytesize.desc: + """A string that represents a number of bytes, for example: 10B, 640kb, 4MB, 1GB. Units are interpreted as powers of 1024, and the unit part is case-insensitive.""" + + secret.desc: + """A string holding some sensitive information, such as a password. When secret starts with file://, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value.""" + +} diff --git a/scripts/schema-dump-reformat.escript b/scripts/schema-dump-reformat.escript index 31cfdd7d9..a3e059b70 100755 --- a/scripts/schema-dump-reformat.escript +++ b/scripts/schema-dump-reformat.escript @@ -18,7 +18,8 @@ main(_) -> halt(1). reformat(Json) -> - emqx_conf:reformat_schema_dump(fix(Json)). + %% NOTE: Assuming schema would contain no types needing localized typedocs. + emqx_conf:reformat_schema_dump(fix(Json), _Lang = "en"). %% fix old type specs to make them compatible with new type specs fix(#{ From da49909ac45e0f2946d18eedb5e806324b3a6655 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 21 Dec 2023 13:42:30 +0100 Subject: [PATCH 5/8] chore(typedoc): refine some descriptions Co-authored-by: Zaiming (Stone) Shi --- rel/i18n/emqx_conf_schema_types.hocon | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rel/i18n/emqx_conf_schema_types.hocon b/rel/i18n/emqx_conf_schema_types.hocon index 4955d3173..6b9dac9ea 100644 --- a/rel/i18n/emqx_conf_schema_types.hocon +++ b/rel/i18n/emqx_conf_schema_types.hocon @@ -1,12 +1,12 @@ emqx_conf_schema_types { duration.desc: - """A string that represents a time duration, for example: 10s, 2.5m, 1h30m, 1W2D, or 2345ms, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing 1200ms for Duration(s) is equivalent to writing 1s. It doesn't matter if units are in upper or lower case.""" + """A string that represents a time duration, for example: 10s, 2.5m, 1h30m, 1W2D, or 2345ms, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing 1200ms for Duration(s) is equivalent to writing 1s. The unit part is case-insensitive.""" bytesize.desc: """A string that represents a number of bytes, for example: 10B, 640kb, 4MB, 1GB. Units are interpreted as powers of 1024, and the unit part is case-insensitive.""" secret.desc: - """A string holding some sensitive information, such as a password. When secret starts with file://, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value.""" + """A string holding some sensitive information, such as a password. When secret starts with file://, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value. Note: when clustered, all EMQX nodes should have the same file present before using file:// secrets.""" } From 1290f1794abe49c6e7ca2449c3f01ec4edb835e3 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 21 Dec 2023 13:44:25 +0100 Subject: [PATCH 6/8] chore: drop `schema-dump-reformat.escript` It should not be needed anymore. --- scripts/schema-dump-reformat.escript | 133 --------------------------- 1 file changed, 133 deletions(-) delete mode 100755 scripts/schema-dump-reformat.escript diff --git a/scripts/schema-dump-reformat.escript b/scripts/schema-dump-reformat.escript deleted file mode 100755 index a3e059b70..000000000 --- a/scripts/schema-dump-reformat.escript +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env escript - -%% This script translates the hocon_schema_json's schema dump to a new format. -%% It is used to convert older version EMQX's schema dumps to the new format -%% after all files are upgraded to the new format, this script can be removed. - --mode(compile). - -main([Input]) -> - ok = add_libs(), - _ = atoms(), - {ok, Data} = file:read_file(Input), - Json = jsx:decode(Data), - NewJson = reformat(Json), - io:format("~s~n", [jsx:encode(NewJson)]); -main(_) -> - io:format("Usage: schema-dump-reformat.escript ~n"), - halt(1). - -reformat(Json) -> - %% NOTE: Assuming schema would contain no types needing localized typedocs. - emqx_conf:reformat_schema_dump(fix(Json), _Lang = "en"). - -%% fix old type specs to make them compatible with new type specs -fix(#{ - <<"kind">> := <<"union">>, - <<"members">> := [#{<<"name">> := <<"string()">>}, #{<<"name">> := <<"function()">>}] -}) -> - %% s3_exporter.secret_access_key - #{ - kind => primitive, - name => <<"string()">> - }; -fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_conf_schema:log_level()">>}) -> - #{ - kind => enum, - symbols => [emergency, alert, critical, error, warning, notice, info, debug, none, all] - }; -fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_connector_http:pool_type()">>}) -> - #{kind => enum, symbols => [random, hash]}; -fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_bridge_http_connector:pool_type()">>}) -> - #{kind => enum, symbols => [random, hash]}; -fix(Map) when is_map(Map) -> - maps:from_list(fix(maps:to_list(Map))); -fix(List) when is_list(List) -> - lists:map(fun fix/1, List); -fix({<<"kind">>, Kind}) -> - {kind, binary_to_atom(Kind, utf8)}; -fix({<<"name">>, Type}) -> - {name, fix_type(Type)}; -fix({K, V}) -> - {binary_to_atom(K, utf8), fix(V)}; -fix(V) when is_number(V) -> - V; -fix(V) when is_atom(V) -> - V; -fix(V) when is_binary(V) -> - V. - -%% ensure below ebin dirs are added to code path: -%% _build/default/lib/*/ebin -%% _build/emqx/lib/*/ebin -%% _build/emqx-enterprise/lib/*/ebin -add_libs() -> - Profile = os:getenv("PROFILE"), - case Profile of - "emqx" -> - ok; - "emqx-enterprise" -> - ok; - _ -> - io:format("PROFILE is not set~n"), - halt(1) - end, - Dirs = - filelib:wildcard("_build/default/lib/*/ebin") ++ - filelib:wildcard("_build/" ++ Profile ++ "/lib/*/ebin"), - lists:foreach(fun add_lib/1, Dirs). - -add_lib(Dir) -> - code:add_patha(Dir), - Beams = filelib:wildcard(Dir ++ "/*.beam"), - _ = spawn(fun() -> lists:foreach(fun load_beam/1, Beams) end), - ok. - -load_beam(BeamFile) -> - ModuleName = filename:basename(BeamFile, ".beam"), - Module = list_to_atom(ModuleName), - %% load the beams to make sure the atoms are existing - code:ensure_loaded(Module), - ok. - -fix_type(<<"[{string(), string()}]">>) -> - <<"map()">>; -fix_type(<<"[{binary(), binary()}]">>) -> - <<"map()">>; -fix_type(<<"emqx_limiter_schema:rate()">>) -> - <<"string()">>; -fix_type(<<"emqx_limiter_schema:burst_rate()">>) -> - <<"string()">>; -fix_type(<<"emqx_limiter_schema:capacity()">>) -> - <<"string()">>; -fix_type(<<"emqx_limiter_schema:initial()">>) -> - <<"string()">>; -fix_type(<<"emqx_limiter_schema:failure_strategy()">>) -> - <<"string()">>; -fix_type(<<"emqx_conf_schema:file()">>) -> - <<"string()">>; -fix_type(<<"#{term() => binary()}">>) -> - <<"map()">>; -fix_type(<<"[term()]">>) -> - %% jwt claims - <<"map()">>; -fix_type(<<"emqx_ee_bridge_influxdb:write_syntax()">>) -> - <<"string()">>; -fix_type(<<"emqx_bridge_influxdb:write_syntax()">>) -> - <<"string()">>; -fix_type(<<"emqx_schema:mqtt_max_packet_size()">>) -> - <<"non_neg_integer()">>; -fix_type(<<"emqx_s3_schema:secret_access_key()">>) -> - <<"string()">>; -fix_type(Type) -> - Type. - -%% ensure atoms are loaded -%% these atoms are from older version of emqx -atoms() -> - [ - emqx_ee_connector_clickhouse, - emqx_ee_bridge_gcp_pubsub, - emqx_ee_bridge_influxdb, - emqx_connector_http - ]. From aa3f8d6735cae1d9876627fd6e973ae0fb2a8f08 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Thu, 21 Dec 2023 17:13:37 +0100 Subject: [PATCH 7/8] fix(typedoc): meld it into the field doc in the meantime --- apps/emqx_conf/src/emqx_conf.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/emqx_conf/src/emqx_conf.erl b/apps/emqx_conf/src/emqx_conf.erl index 4f845a166..0a8339ddd 100644 --- a/apps/emqx_conf/src/emqx_conf.erl +++ b/apps/emqx_conf/src/emqx_conf.erl @@ -323,10 +323,12 @@ format_fields(Fields, DescResolver) -> [format_field(F, DescResolver) || F <- Fields]. format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver) -> + TypeDoc = format_type_desc(Type, DescResolver), L = [ {text, Name}, {type, format_type(Type)}, - {typedoc, format_type_desc(Type, DescResolver)}, + %% TODO: Make it into a separate field. + %% {typedoc, format_type_desc(Type, DescResolver)}, {refs, format_refs(Type)}, {aliases, case Aliases of @@ -334,7 +336,7 @@ format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver _ -> Aliases end}, {default, maps:get(hocon, maps:get(default, F, #{}), undefined)}, - {doc, maps:get(desc, F, undefined)} + {doc, join_format([maps:get(desc, F, undefined), TypeDoc])} ], maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]). @@ -577,6 +579,14 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) -> typename_to_spec(TypeStr, Module) -> emqx_conf_schema_types:readable_dashboard(Module, TypeStr). +join_format(Snippets) -> + case [S || S <- Snippets, S =/= undefined] of + [] -> + undefined; + NonEmpty -> + to_bin(lists:join("
", NonEmpty)) + end. + to_bin(List) when is_list(List) -> iolist_to_binary(List); to_bin(Boolean) when is_boolean(Boolean) -> Boolean; to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8); From 36d29295341215041debd2cec11f1d6e40285bce Mon Sep 17 00:00:00 2001 From: "Zaiming (Stone) Shi" Date: Thu, 21 Dec 2023 18:48:09 +0100 Subject: [PATCH 8/8] chore: prepare for release version 5.4.0 --- apps/emqx/include/emqx_release.hrl | 4 +- changes/e5.4.0.en.md | 120 +++++++++++++++++++++++ changes/v5.4.0.en.md | 82 ++++++++++++++++ deploy/charts/emqx-enterprise/Chart.yaml | 4 +- deploy/charts/emqx/Chart.yaml | 4 +- 5 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 changes/e5.4.0.en.md create mode 100644 changes/v5.4.0.en.md diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index ea90bf5dd..c96f93d8e 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,10 +32,10 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.4.0-rc.1"). +-define(EMQX_RELEASE_CE, "5.4.0"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.4.0-rc.1"). +-define(EMQX_RELEASE_EE, "5.4.0"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/changes/e5.4.0.en.md b/changes/e5.4.0.en.md new file mode 100644 index 000000000..d3eaa0b7e --- /dev/null +++ b/changes/e5.4.0.en.md @@ -0,0 +1,120 @@ +# e5.4.0 + +## Breaking Changes + +- [#11994](https://github.com/emqx/emqx/pull/11994) Stop releasing packages for Windows. + +- [#11998](https://github.com/emqx/emqx/pull/11998) Stop releasing packages for MacOS 11 (BigSur). + +- [#12112](https://github.com/emqx/emqx/pull/12112) Stop supporting UDP multicast based clustering strategy. + +- [#10976](https://github.com/emqx/emqx/pull/10976) Fix topic-filter overlapping handling in shared subscription. + * Hook callback `session.subscribed` and `client.subscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from `subopts`. + * Hook callback `session.unsubscribed` and `client.unsubscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#` instead of just `topic1/#`. + * ExHook Proto changed. The `share` field in message `SubOpts` was deprecated. + ExHook Server will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from message `SubOpts`. + * `session.subscribed` and `session.unsubscribed` rule-engine events will have shared subscriptions in their full representation for `topic`, e.g. `$share/group1/topic1/#` instead of just `topic1/#`. + +## Enhancements + +- [#11884](https://github.com/emqx/emqx/pull/11884) Modified the Prometheus API and configuration to implement the following improvements: + + - Restructured configuration sections to group related settings, improving readability and maintainability. + - Introduced `enable_basic_auth` configuration for basic authentication on the scrape API endpoint, enhancing security. + - Maintained backwards compatibility while refactoring code, avoiding breaking changes. + +- [#11896](https://github.com/emqx/emqx/pull/11896) Introduced an enhancement for configuring sensitive authentication fields in bridges, such as passwords, tokens, and secret keys. This improvement allows the use of secrets stored as files in the file system. These secrets can be securely referenced in configuration files using the special `file://` prefix, enhancing the security of sensitive data handling in bridge configurations. + +- [#11921](https://github.com/emqx/emqx/pull/11921) Introduced Open Telemetry Logs Handler that allows to format log events in alignment with the Open Telemetry log data model. This handler facilitates the exportation of formatted log events to a configured Open Telemetry collector or back-end, thereby enhancing log management and integration capabilities. + +- [#11935](https://github.com/emqx/emqx/pull/11935) Switched to the new `v2` routing store schema by default. New schema improves both subscription and routing performance, especially in scenarios with concurrent subscriptions to topic filters sharing common wildcard prefixes. However, it does come with a minor increase in memory usage. This schema also eliminates the need for a separate index, thus inconsistencies in the routing state rarely encountered in previous versions should no longer be possible. + + If a cluster is rolling upgraded from older version, the cluster will continue to use `v1` store until a full cluster (non-rolling) restart happens. + + Users can still opt for the previous schema by configuring the `broker.routing.storage_schema` option to `v1`. However, this also requires a complete, non-rolling restart of the cluster to take effect. + +- [#11984](https://github.com/emqx/emqx/pull/11984) Implemented Open Telemetry distributed tracing feature. + +- [#12017](https://github.com/emqx/emqx/pull/12017) Implemented a dedicated HTTP API for the import and export of configuration and user data. + +- [#12040](https://github.com/emqx/emqx/pull/12040) Upgraded QUIC protocol stack. + +- [#11766](https://github.com/emqx/emqx/pull/11766) Implemented a preliminary Role-Based Access Control for the REST API. In this version, there are three predefined roles: + - Administrator: This role can access all resources. + - Viewer: This role can only view resources and data, corresponding to all GET requests in the REST API. + - Publisher: Specifically tailored for MQTT message publishing, this role is confined to accessing endpoints related to message publication. + +- [#12201](https://github.com/emqx/emqx/pull/11994) Support hot update of TCP/SSL/WS/WSS MQTT listeners configuration. + This allows changing most of the configuration parameters without restarting the listener and disconnecting the clients. The limitations are: + - For TCP/SSL listeners, changes to the following parameters still require listener restart and clients reconnect: + * `bind` + * `tcp_options.backlog` + - For WS/WSS (WebSocket) listeners, changing transport related parameters (listed below) will cause listening socket to be re-opened, but established connections will stay uninterrupted. + * `bind` + * `tcp_options.*` + * `ssl_options.*` + +- [#11608](https://github.com/emqx/emqx/pull/11608) Integrated LDAP bind operation as a new authenticator, providing a more flexible and secure method for user authentication. + +- [#11773](https://github.com/emqx/emqx/pull/11773) Implemented Dashboard support for audit log management. Users can utilize this page to view all change operations performed on EMQX devices and data, such as kicking out devices, creating/deleting rules, etc. + +- [#11778](https://github.com/emqx/emqx/pull/11778) Integrated Microsoft Entra Identity (formerly known as Azure Active Directory) support into the SAML single sign-on (SSO) process. + + +- [#11811](https://github.com/emqx/emqx/pull/11811) Improved the format for the REST API key bootstrap file to support initializing key with a role. + + The new form is:`api_key:api_secret:role`. + + `role` is optional and its default value is `administrator`. + +- [#11852](https://github.com/emqx/emqx/pull/11852) Introduced a new GB/T 32960 gateway, enabling vehicles to connect with EMQX via the GBT32960 vehicular networking protocol. + +- [#11883](https://github.com/emqx/emqx/pull/11883) Introduced a new JT/T808 gateway, enabling vehicles to connect with EMQX via the JT/T 808 vehicular networking protocol. + +- [#11885](https://github.com/emqx/emqx/pull/11885) Introduced a new OCPP gateway for Electric vehicle (EV) charging stations to access EMQX through the OCPP (Open Charge Point Protocol). + +- [#11971](https://github.com/emqx/emqx/pull/11971) Made `/api/v5/load_rebalance/availability_check` public, meaning it no longer requires authentication. This change simplifies the setup of load balancers. + + It improved the gracefulness of the rebalance/evacuation process during the wait health check phase. The connections to nodes marked for eviction are now not prohibited during this phase. + During this phase it is unknown whether these nodes are all marked unhealthy by the load balancer, so prohibiting connections to them may cause multiple unsuccessful reconnection attempts. + +- [#12013](https://github.com/emqx/emqx/pull/12013) The data bridging design has been adjusted to split it into connectors and actions (Sinks). Connectors are used to manage the integration of data with external systems and can be reused across multiple actions, while actions are used to configure how data is processed. This design provides greater flexibility and scalability, resulting in clearer data integration configuration and management. + + The adjusted data bridges includes PostgreSQL, Timescale, and Matrix, which have now been split into connectors and actions APIs, but they remain backward compatible with the old data bridge API. + +- [#12016](https://github.com/emqx/emqx/pull/12016) Enhanced license key management. + + EMQX can now load the license key from a specified file. This is enabled by setting the `license.key` configuration to a file path, which should be prefixed with `"file://"`. + Also added the ability to revert to the default trial license by setting `license.key = default`. This option simplifies the process of returning to the trial license if needed. + +- [#12129](https://github.com/emqx/emqx/pull/12129) Default license renewal. Replaced old license issued in Jan 2023. New license supports up to 25 concurrent connections. + +### Bug Fixes + +- [#10976](https://github.com/emqx/emqx/pull/10976) Fixed topic-filter overlapping handling in shared subscription. + In the previous implementation, the storage method for subscription options did not provide adequate support for shared subscriptions. This resulted in message routing failures and leakage of routing tables between nodes during the "subscribe-unsubscribe" process with specific order and topics. + +- [#12048](https://github.com/emqx/emqx/pull/12048) Fixed COAP gateway bug that caused it to ignore subscription options. + +- [#12078](https://github.com/emqx/emqx/pull/12078) Upgraded grpc-erl to 0.6.12. This update addresses a potential deadlock issue where the grpc client started dependent apps lazily. + +- [#12081](https://github.com/emqx/emqx/pull/12081) Updated `gen_rpc` library to version 3.3.1. The new version includes several performance improvements: + + - Avoiding allocating extra memory for the packets before they are sent to the wire in some cases. + + - Bypassing network for the local calls. + + - Avoid senstive data leaking in debug logs [#12202](https://github.com/emqx/emqx/pull/12202) + +- [#12111](https://github.com/emqx/emqx/pull/12111) Fixed an issue when API tokens were sometimes unavailable immediately after login due to race condition. + +- [#12121](https://github.com/emqx/emqx/pull/12121) Fixed an issue where nodes in the cluster would occasionally return a stale view when updating configurations on different nodes concurrently. + +- [#12158](https://github.com/emqx/emqx/pull/12158) Fixed an issue when the rule engine cannot connect to Redis hosted by Upstash. + + Before the fix, after establishing a TCP connection with the Redis service, the Redis driver of EMQX used [Inline Commands](https://redis.io/docs/reference/protocol-spec/#inline-commands) to send AUTH and SELECT commands. However, the `upstash` Redis service does not support Inline Commands, which causes the rule engine to fail to connect to the `upstash` Redis service. + After the fix, the Redis driver of EMQX uses RESP (REdis Serialization Protocol) to send AUTH and SELECT commands. + +- [#12176](https://github.com/emqx/emqx/pull/12176) Always acknowledge `DISCONNECT` packet to MQTT-SN client regardless of whether the connection has been successfully established before. + +- [#12180](https://github.com/emqx/emqx/pull/12180) Fix an issue where DTLS enabled MQTT-SN gateways could not be started, caused by incompatibility of default listener configuration with the DTLS implementation. diff --git a/changes/v5.4.0.en.md b/changes/v5.4.0.en.md new file mode 100644 index 000000000..c0704307a --- /dev/null +++ b/changes/v5.4.0.en.md @@ -0,0 +1,82 @@ +# v5.4.0 + +## Breaking Changes + +- [#11994](https://github.com/emqx/emqx/pull/11994) Stop releasing packages for Windows. + +- [#11998](https://github.com/emqx/emqx/pull/11998) Stop releasing packages for MacOS 11 (BigSur). + +- [#12112](https://github.com/emqx/emqx/pull/12112) Stop supporting UDP multicast based clustering strategy. + +- [#10976](https://github.com/emqx/emqx/pull/10976) Fix topic-filter overlapping handling in shared subscription. + * Hook callback `session.subscribed` and `client.subscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from `subopts`. + * Hook callback `session.unsubscribed` and `client.unsubscribe` will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#` instead of just `topic1/#`. + * ExHook Proto changed. The `share` field in message `SubOpts` was deprecated. + ExHook Server will now receive shared subscription in its full representation, e.g. `$share/group1/topic1/#`, and the `share` property is deleted from message `SubOpts`. + * `session.subscribed` and `session.unsubscribed` rule-engine events will have shared subscriptions in their full representation for `topic`, e.g. `$share/group1/topic1/#` instead of just `topic1/#`. + +## Enhancements + +- [#11884](https://github.com/emqx/emqx/pull/11884) Modified the Prometheus API and configuration to implement the following improvements: + + - Restructured configuration sections to group related settings, improving readability and maintainability. + - Introduced `enable_basic_auth` configuration for basic authentication on the scrape API endpoint, enhancing security. + - Maintained backwards compatibility while refactoring code, avoiding breaking changes. + +- [#11896](https://github.com/emqx/emqx/pull/11896) Introduced an enhancement for configuring sensitive authentication fields in bridges, such as passwords, tokens, and secret keys. This improvement allows the use of secrets stored as files in the file system. These secrets can be securely referenced in configuration files using the special `file://` prefix, enhancing the security of sensitive data handling in bridge configurations. + +- [#11921](https://github.com/emqx/emqx/pull/11921) Introduced Open Telemetry Logs Handler that allows to format log events in alignment with the Open Telemetry log data model. This handler facilitates the exportation of formatted log events to a configured Open Telemetry collector or back-end, thereby enhancing log management and integration capabilities. + +- [#11935](https://github.com/emqx/emqx/pull/11935) Switched to the new `v2` routing store schema by default. New schema improves both subscription and routing performance, especially in scenarios with concurrent subscriptions to topic filters sharing common wildcard prefixes. However, it does come with a minor increase in memory usage. This schema also eliminates the need for a separate index, thus inconsistencies in the routing state rarely encountered in previous versions should no longer be possible. + + If a cluster is rolling upgraded from older version, the cluster will continue to use `v1` store until a full cluster (non-rolling) restart happens. + + Users can still opt for the previous schema by configuring the `broker.routing.storage_schema` option to `v1`. However, this also requires a complete, non-rolling restart of the cluster to take effect. + +- [#11984](https://github.com/emqx/emqx/pull/11984) Implemented Open Telemetry distributed tracing feature. + +- [#12017](https://github.com/emqx/emqx/pull/12017) Implemented a dedicated HTTP API for the import and export of configuration and user data. + +- [#12040](https://github.com/emqx/emqx/pull/12040) Upgraded QUIC protocol stack. + +- [#11766](https://github.com/emqx/emqx/pull/11766) Implemented a preliminary Role-Based Access Control for the REST API. In this version, there are three predefined roles: + - Administrator: This role can access all resources. + - Viewer: This role can only view resources and data, corresponding to all GET requests in the REST API. + - Publisher: Specifically tailored for MQTT message publishing, this role is confined to accessing endpoints related to message publication. + +- [#12201](https://github.com/emqx/emqx/pull/11994) Support hot update of TCP/SSL/WS/WSS MQTT listeners configuration. + This allows changing most of the configuration parameters without restarting the listener and disconnecting the clients. The limitations are: + - For TCP/SSL listeners, changes to the following parameters still require listener restart and clients reconnect: + * `bind` + * `tcp_options.backlog` + - For WS/WSS (WebSocket) listeners, changing transport related parameters (listed below) will cause listening socket to be re-opened, but established connections will stay uninterrupted. + * `bind` + * `tcp_options.*` + * `ssl_options.*` + +## Bug Fixes + +- [#12048](https://github.com/emqx/emqx/pull/12048) Fixed COAP gateway bug that caused it to ignore subscription options. + +- [#12078](https://github.com/emqx/emqx/pull/12078) Upgraded grpc-erl to 0.6.12. This update addresses a potential deadlock issue where the grpc client started dependent apps lazily. + +- [#12081](https://github.com/emqx/emqx/pull/12081) Updated `gen_rpc` library to version 3.3.1. The new version includes several performance improvements: + + - Avoiding allocating extra memory for the packets before they are sent to the wire in some cases. + + - Bypassing network for the local calls. + + - Avoid senstive data leaking in debug logs [#12202](https://github.com/emqx/emqx/pull/12202) + +- [#12111](https://github.com/emqx/emqx/pull/12111) Fixed an issue when API tokens were sometimes unavailable immediately after login due to race condition. + +- [#12121](https://github.com/emqx/emqx/pull/12121) Fixed an issue where nodes in the cluster would occasionally return a stale view when updating configurations on different nodes concurrently. + +- [#12158](https://github.com/emqx/emqx/pull/12158) Fixed an issue when the rule engine cannot connect to Redis hosted by Upstash. + + Before the fix, after establishing a TCP connection with the Redis service, the Redis driver of EMQX used [Inline Commands](https://redis.io/docs/reference/protocol-spec/#inline-commands) to send AUTH and SELECT commands. However, the `upstash` Redis service does not support Inline Commands, which causes the rule engine to fail to connect to the `upstash` Redis service. + After the fix, the Redis driver of EMQX uses RESP (REdis Serialization Protocol) to send AUTH and SELECT commands. + +- [#12176](https://github.com/emqx/emqx/pull/12176) Always acknowledge `DISCONNECT` packet to MQTT-SN client regardless of whether the connection has been successfully established before. + +- [#12180](https://github.com/emqx/emqx/pull/12180) Fix an issue where DTLS enabled MQTT-SN gateways could not be started, caused by incompatibility of default listener configuration with the DTLS implementation. diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index c60246f69..acbcaffb2 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.4.0-rc.1 +version: 5.4.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.4.0-rc.1 +appVersion: 5.4.0 diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 7d2982c39..41ee9a4cb 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.4.0-rc.1 +version: 5.4.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.4.0-rc.1 +appVersion: 5.4.0