From 1c86bd6199ebd019a2ad7b35d2e6313df25ef4bd Mon Sep 17 00:00:00 2001 From: Shawn <506895667@qq.com> Date: Wed, 11 Aug 2021 21:29:03 +0800 Subject: [PATCH] feat(emqx_config): add emqx_config:fill_defaults/1 --- apps/emqx/rebar.config | 2 +- apps/emqx/src/emqx_config.erl | 128 +++++++++++++++--- apps/emqx/src/emqx_map_lib.erl | 45 ++++-- apps/emqx_machine/src/emqx_machine_schema.erl | 1 + rebar.config | 2 +- 5 files changed, 147 insertions(+), 31 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index ebe46559b..f34173fae 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -15,7 +15,7 @@ , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.4"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.1"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} diff --git a/apps/emqx/src/emqx_config.erl b/apps/emqx/src/emqx_config.erl index 151cf8e8e..dc1c6d7c5 100644 --- a/apps/emqx/src/emqx_config.erl +++ b/apps/emqx/src/emqx_config.erl @@ -20,14 +20,20 @@ -export([ init_load/2 , read_override_conf/0 , check_config/2 + , fill_defaults/1 + , fill_defaults/2 , save_configs/4 , save_to_app_env/1 , save_to_config_map/2 , save_to_override_conf/1 ]). --export([get_root/1, - get_root_raw/1]). +-export([ get_root/1 + , get_root_raw/1 + ]). + +-export([ get_default_value/1 + ]). -export([ get/1 , get/2 @@ -37,6 +43,12 @@ , put/2 ]). +-export([ save_schema_mod/1 + , get_schema_mod/0 + , get_schema_mod/1 + , get_root_names/0 + ]). + -export([ get_zone_conf/2 , get_zone_conf/3 , put_zone_conf/3 @@ -53,6 +65,7 @@ , update/3 , remove/1 , remove/2 + , reset/1 ]). -export([ get_raw/1 @@ -63,6 +76,7 @@ -define(CONF, conf). -define(RAW_CONF, raw_conf). +-define(PERSIS_MOD_ROOTNAMES, {?MODULE, default_conf}). -define(PERSIS_KEY(TYPE, ROOT), {?MODULE, TYPE, ROOT}). -define(ZONE_CONF_PATH(ZONE, PATH), [zones, ZONE | PATH]). -define(LISTENER_CONF_PATH(ZONE, LISTENER, PATH), [zones, ZONE, listeners, LISTENER | PATH]). @@ -170,20 +184,49 @@ put(KeyPath, Config) -> do_put(?CONF, KeyPath, Config). -spec update(emqx_map_lib:config_key_path(), update_request()) -> ok | {error, term()}. -update(KeyPath, UpdateReq) -> - update(emqx_schema, KeyPath, UpdateReq). +update([RootName | _] = KeyPath, UpdateReq) -> + update(get_schema_mod(RootName), KeyPath, UpdateReq). -spec update(module(), emqx_map_lib:config_key_path(), update_request()) -> ok | {error, term()}. -update(SchemaModule, KeyPath, UpdateReq) -> - emqx_config_handler:update_config(SchemaModule, KeyPath, {update, UpdateReq}). +update(SchemaMod, KeyPath, UpdateReq) -> + emqx_config_handler:update_config(SchemaMod, KeyPath, {update, UpdateReq}). -spec remove(emqx_map_lib:config_key_path()) -> ok | {error, term()}. -remove(KeyPath) -> - remove(emqx_schema, KeyPath). +remove([RootName | _] = KeyPath) -> + remove(get_schema_mod(RootName), KeyPath). -remove(SchemaModule, KeyPath) -> - emqx_config_handler:update_config(SchemaModule, KeyPath, remove). +remove(SchemaMod, KeyPath) -> + emqx_config_handler:update_config(SchemaMod, KeyPath, remove). + +-spec reset(emqx_map_lib:config_key_path()) -> ok | {error, term()}. +reset([RootName | _] = KeyPath) -> + case get_default_value(KeyPath) of + {ok, Default} -> + emqx_config_handler:update_config(get_schema_mod(RootName), KeyPath, + {update, Default}); + {error, _} = Error -> + Error + end. + +-spec get_default_value(emqx_map_lib:config_key_path()) -> ok | {error, term()}. +get_default_value([RootName | _] = KeyPath) -> + BinKeyPath = [bin(Key) || Key <- KeyPath], + case find_raw([RootName]) of + {ok, RawConf} -> + RawConf1 = emqx_map_lib:deep_remove(BinKeyPath, RawConf), + SchemaMod = get_schema_mod(RootName), + try fill_defaults(SchemaMod, RawConf1) of FullConf -> + case emqx_map_lib:deep_find(BinKeyPath, FullConf) of + {not_found, _, _} -> {error, no_default_value}; + {ok, Val} -> {ok, Val} + end + catch error:_ -> + {error, required_conf} + end; + {not_found, _, _} -> + {error, {rootname_not_found, RootName}} + end. -spec get_raw(emqx_map_lib:config_key_path()) -> term(). get_raw(KeyPath) -> do_get(?RAW_CONF, KeyPath). @@ -208,7 +251,7 @@ put_raw(KeyPath, Config) -> do_put(?RAW_CONF, KeyPath, Config). %% NOTE: The order of the files is significant, configs from files orderd %% in the rear of the list overrides prior values. -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok. -init_load(SchemaModule, Conf) when is_list(Conf) orelse is_binary(Conf) -> +init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) -> ParseOptions = #{format => richmap}, Parser = case is_binary(Conf) of true -> fun hocon:binary/2; @@ -216,39 +259,78 @@ init_load(SchemaModule, Conf) when is_list(Conf) orelse is_binary(Conf) -> end, case Parser(Conf, ParseOptions) of {ok, RawRichConf} -> - init_load(SchemaModule, RawRichConf); + init_load(SchemaMod, RawRichConf); {error, Reason} -> logger:error(#{msg => failed_to_load_hocon_conf, reason => Reason }), error(failed_to_load_hocon_conf) end; -init_load(SchemaModule, RawRichConf) when is_map(RawRichConf) -> +init_load(SchemaMod, RawRichConf) when is_map(RawRichConf) -> %% check with richmap for line numbers in error reports (future enhancement) Opts = #{return_plain => true, nullable => true }, %% this call throws exception in case of check failure - {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaModule, RawRichConf, Opts), - ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(CheckedConf), - hocon_schema:richmap_to_map(RawRichConf)). + {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaMod, RawRichConf, Opts), + ok = save_schema_mod(SchemaMod), + ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(normalize_conf(CheckedConf)), + normalize_conf(hocon_schema:richmap_to_map(RawRichConf))). + +normalize_conf(Conf) -> + maps:with(get_root_names(), Conf). -spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf} when AppEnvs :: app_envs(), CheckedConf :: config(). -check_config(SchemaModule, RawConf) -> +check_config(SchemaMod, RawConf) -> Opts = #{return_plain => true, nullable => true, format => map }, {AppEnvs, CheckedConf} = - hocon_schema:map_translate(SchemaModule, RawConf, Opts), + hocon_schema:map_translate(SchemaMod, RawConf, Opts), Conf = maps:with(maps:keys(RawConf), CheckedConf), {AppEnvs, emqx_map_lib:unsafe_atom_key_map(Conf)}. +-spec fill_defaults(raw_config()) -> map(). +fill_defaults(RawConf) -> + RootNames = get_root_names(), + maps:fold(fun(Key, Conf, Acc) -> + SubMap = #{Key => Conf}, + WithDefaults = case lists:member(Key, RootNames) of + true -> fill_defaults(get_schema_mod(Key), SubMap); + false -> SubMap + end, + maps:merge(Acc, WithDefaults) + end, #{}, RawConf). + +-spec fill_defaults(module(), raw_config()) -> map(). +fill_defaults(SchemaMod, RawConf) -> + hocon_schema:check_plain(SchemaMod, RawConf, + #{nullable => true, no_conversion => true}, [str(K) || K <- maps:keys(RawConf)]). + -spec read_override_conf() -> raw_config(). read_override_conf() -> load_hocon_file(emqx_override_conf_name(), map). +-spec save_schema_mod(module()) -> ok. +save_schema_mod(SchemaMod) -> + OldMods = get_schema_mod(), + NewMods = maps:from_list([{bin(RootName), SchemaMod} || RootName <- SchemaMod:structs()]), + persistent_term:put(?PERSIS_MOD_ROOTNAMES, maps:merge(OldMods, NewMods)). + +-spec get_schema_mod() -> #{binary() => atom()}. +get_schema_mod() -> + persistent_term:get(?PERSIS_MOD_ROOTNAMES, #{}). + +-spec get_schema_mod(atom() | binary()) -> [module()]. +get_schema_mod(RootName) -> + maps:get(bin(RootName), get_schema_mod()). + +-spec get_root_names() -> [binary()]. +get_root_names() -> + maps:keys(get_schema_mod()). + -spec save_configs(app_envs(), config(), raw_config(), raw_config()) -> ok | {error, term()}. save_configs(_AppEnvs, Conf, RawConf, OverrideConf) -> %% We may need also support hot config update for the apps that use application envs. @@ -338,10 +420,20 @@ do_deep_put(?RAW_CONF, KeyPath, Map, Value) -> atom(Bin) when is_binary(Bin) -> binary_to_existing_atom(Bin, latin1); +atom(Str) when is_list(Str) -> + list_to_existing_atom(Str); atom(Atom) when is_atom(Atom) -> Atom. +str(Bin) when is_binary(Bin) -> + binary_to_list(Bin); +str(Str) when is_list(Str) -> + Str; +str(Atom) when is_atom(Atom) -> + atom_to_list(Atom). + bin(Bin) when is_binary(Bin) -> Bin; +bin(Str) when is_list(Str) -> list_to_binary(Str); bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8). conf_key(?CONF, RootName) -> diff --git a/apps/emqx/src/emqx_map_lib.erl b/apps/emqx/src/emqx_map_lib.erl index d720e771e..cda4f3a85 100644 --- a/apps/emqx/src/emqx_map_lib.erl +++ b/apps/emqx/src/emqx_map_lib.erl @@ -23,6 +23,8 @@ , deep_merge/2 , safe_atom_key_map/1 , unsafe_atom_key_map/1 + , jsonable_map/1 + , deep_convert/2 ]). -export_type([config_key/0, config_key_path/0]). @@ -97,21 +99,42 @@ deep_merge(BaseMap, NewMap) -> end, #{}, BaseMap), maps:merge(MergedBase, maps:with(NewKeys, NewMap)). +-spec deep_convert(map(), fun((K::any(), V::any()) -> {K1::any(), V1::any()})) -> map(). +deep_convert(Map, ConvFun) when is_map(Map) -> + maps:fold(fun(K, V, Acc) -> + {K1, V1} = ConvFun(K, deep_convert(V, ConvFun)), + Acc#{K1 => V1} + end, #{}, Map); +deep_convert(ListV, ConvFun) when is_list(ListV) -> + [deep_convert(V, ConvFun) || V <- ListV]; +deep_convert(Val, _) -> Val. + +-spec unsafe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}. unsafe_atom_key_map(Map) -> covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end). +-spec safe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}. safe_atom_key_map(Map) -> covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end). +-spec jsonable_map(map()) -> map(). +jsonable_map(Map) -> + deep_convert(Map, fun(K, V) -> + {jsonable_value(K), jsonable_value(V)} + end). + +jsonable_value([]) -> []; +jsonable_value(Val) when is_list(Val) -> + case io_lib:printable_unicode_list(Val) of + true -> unicode:characters_to_binary(Val); + false -> Val + end; +jsonable_value(Val) -> + Val. + %%--------------------------------------------------------------------------- -covert_keys_to_atom(BinKeyMap, Conv) when is_map(BinKeyMap) -> - maps:fold( - fun(K, V, Acc) when is_binary(K) -> - Acc#{Conv(K) => covert_keys_to_atom(V, Conv)}; - (K, V, Acc) when is_atom(K) -> - %% richmap keys - Acc#{K => covert_keys_to_atom(V, Conv)} - end, #{}, BinKeyMap); -covert_keys_to_atom(ListV, Conv) when is_list(ListV) -> - [covert_keys_to_atom(V, Conv) || V <- ListV]; -covert_keys_to_atom(Val, _) -> Val. +covert_keys_to_atom(BinKeyMap, Conv) -> + deep_convert(BinKeyMap, fun + (K, V) when is_atom(K) -> {K, V}; + (K, V) when is_binary(K) -> {Conv(K), V} + end). diff --git a/apps/emqx_machine/src/emqx_machine_schema.erl b/apps/emqx_machine/src/emqx_machine_schema.erl index f0de9d443..cef5e525f 100644 --- a/apps/emqx_machine/src/emqx_machine_schema.erl +++ b/apps/emqx_machine/src/emqx_machine_schema.erl @@ -54,6 +54,7 @@ , emqx_dashboard_schema , emqx_gateway_schema , emqx_prometheus_schema + , emqx_rule_engine_schema , emqx_exhook_schema ]). diff --git a/rebar.config b/rebar.config index b59909dbe..1abaef868 100644 --- a/rebar.config +++ b/rebar.config @@ -61,7 +61,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.1"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.1.0"}}} ]}.