From d640e2ccfa68a1c0529ef302b3eadc627d07f5e5 Mon Sep 17 00:00:00 2001 From: Turtle Date: Thu, 17 Jun 2021 15:11:26 +0800 Subject: [PATCH 01/44] chore: mv emqx_connector to emqx_data_bridge --- apps/emqx_connector/.gitignore | 19 ---------- apps/emqx_connector/README.md | 27 -------------- apps/emqx_connector/etc/emqx_connector.conf | 4 --- .../emqx_connector/priv/emqx_connector.schema | 2 -- apps/emqx_connector/rebar.config | 13 ------- .../emqx_connector/src/emqx_connector.app.src | 17 --------- apps/emqx_connector/src/emqx_connector.erl | 16 --------- .../emqx_connector/src/emqx_connector_app.erl | 31 ---------------- .../emqx_connector/src/emqx_connector_sup.erl | 36 ------------------- .../include/emqx_data_bridge.hrl} | 0 .../priv/emqx_data_bridge.schema | 16 --------- apps/emqx_data_bridge/rebar.config | 1 + .../src/connector}/emqx_connector_ldap.erl | 2 +- .../src/connector}/emqx_connector_mongo.erl | 2 +- .../src/connector}/emqx_connector_mysql.erl | 0 .../src/connector}/emqx_connector_pgsql.erl | 0 .../src/connector}/emqx_connector_redis.erl | 2 +- .../connector}/emqx_connector_schema_lib.erl | 2 +- .../src/emqx_data_bridge.app.src | 3 +- .../emqx_data_bridge/src/emqx_data_bridge.erl | 5 +-- apps/emqx_resource/priv/emqx_resource.schema | 2 -- data/loaded_plugins.tmpl | 3 -- rebar.config.erl | 9 ++--- 23 files changed, 13 insertions(+), 199 deletions(-) delete mode 100644 apps/emqx_connector/.gitignore delete mode 100644 apps/emqx_connector/README.md delete mode 100644 apps/emqx_connector/etc/emqx_connector.conf delete mode 100644 apps/emqx_connector/priv/emqx_connector.schema delete mode 100644 apps/emqx_connector/rebar.config delete mode 100644 apps/emqx_connector/src/emqx_connector.app.src delete mode 100644 apps/emqx_connector/src/emqx_connector.erl delete mode 100644 apps/emqx_connector/src/emqx_connector_app.erl delete mode 100644 apps/emqx_connector/src/emqx_connector_sup.erl rename apps/{emqx_connector/include/emqx_connector.hrl => emqx_data_bridge/include/emqx_data_bridge.hrl} (100%) delete mode 100644 apps/emqx_data_bridge/priv/emqx_data_bridge.schema rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_ldap.erl (99%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_mongo.erl (99%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_mysql.erl (100%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_pgsql.erl (100%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_redis.erl (99%) rename apps/{emqx_connector/src => emqx_data_bridge/src/connector}/emqx_connector_schema_lib.erl (99%) delete mode 100644 apps/emqx_resource/priv/emqx_resource.schema diff --git a/apps/emqx_connector/.gitignore b/apps/emqx_connector/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/apps/emqx_connector/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md deleted file mode 100644 index 879669f93..000000000 --- a/apps/emqx_connector/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# emqx_connector - -This application is a collection of `connectors`. - -A `connector` is a callback module of `emqx_resource` that maintains the data related to -external resources. Put all resource related callback modules in a single application is good as -we can put some util functions/modules here for reusing purpose. - -For example, a mysql connector is an emqx resource that maintains all the mysql connection -related parameters (configs) and the TCP connections to the mysql server. - -An mysql connector can be used as following: - -``` -(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). -[#{config => - #{auto_reconnect => true,cacertfile => [],certfile => [], - database => "mqtt",keyfile => [],password => "public", - pool_size => 1, - server => {{127,0,0,1},3306}, - ssl => false,user => "root",verify => false}, - id => <<"mysql-abc">>,mod => emqx_connector_mysql, - state => #{poolname => 'mysql-abc'}, - status => started}] -(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}). -{ok,[<<"count(1)">>],[[1]]} -``` diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf deleted file mode 100644 index db4402d47..000000000 --- a/apps/emqx_connector/etc/emqx_connector.conf +++ /dev/null @@ -1,4 +0,0 @@ -##-------------------------------------------------------------------- -## EMQ X CONNECTOR Plugin -##-------------------------------------------------------------------- - diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema deleted file mode 100644 index b8476c4d9..000000000 --- a/apps/emqx_connector/priv/emqx_connector.schema +++ /dev/null @@ -1,2 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_connector config mapping diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config deleted file mode 100644 index 53b5c63e8..000000000 --- a/apps/emqx_connector/rebar.config +++ /dev/null @@ -1,13 +0,0 @@ -{erl_opts, [ - nowarn_unused_import, - debug_info -]}. - -{deps, [ - {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}} -]}. - -{shell, [ - % {config, "config/sys.config"}, - {apps, [emqx_connector]} -]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src deleted file mode 100644 index a821b8f13..000000000 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ /dev/null @@ -1,17 +0,0 @@ -{application, emqx_connector, - [{description, "An OTP application"}, - {vsn, "0.1.0"}, - {registered, []}, - {mod, {emqx_connector_app, []}}, - {applications, - [kernel, - stdlib, - emqx_resource, - ecpool - ]}, - {env,[]}, - {modules, []}, - - {licenses, ["Apache 2.0"]}, - {links, []} - ]}. diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl deleted file mode 100644 index dd0359348..000000000 --- a/apps/emqx_connector/src/emqx_connector.erl +++ /dev/null @@ -1,16 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_connector). diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl deleted file mode 100644 index 4bbad75cf..000000000 --- a/apps/emqx_connector/src/emqx_connector_app.erl +++ /dev/null @@ -1,31 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - --module(emqx_connector_app). - --behaviour(application). - --emqx_plugin(?MODULE). - --export([start/2, stop/1]). - -start(_StartType, _StartArgs) -> - emqx_connector_sup:start_link(). - -stop(_State) -> - ok. - -%% internal functions diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl deleted file mode 100644 index 603b9a8ad..000000000 --- a/apps/emqx_connector/src/emqx_connector_sup.erl +++ /dev/null @@ -1,36 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- --module(emqx_connector_sup). - --behaviour(supervisor). - --export([start_link/0]). - --export([init/1]). - --define(SERVER, ?MODULE). - -start_link() -> - supervisor:start_link({local, ?SERVER}, ?MODULE, []). - -init([]) -> - SupFlags = #{strategy => one_for_all, - intensity => 0, - period => 1}, - ChildSpecs = [], - {ok, {SupFlags, ChildSpecs}}. - -%% internal functions diff --git a/apps/emqx_connector/include/emqx_connector.hrl b/apps/emqx_data_bridge/include/emqx_data_bridge.hrl similarity index 100% rename from apps/emqx_connector/include/emqx_connector.hrl rename to apps/emqx_data_bridge/include/emqx_data_bridge.hrl diff --git a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema deleted file mode 100644 index c9cb3f2c0..000000000 --- a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema +++ /dev/null @@ -1,16 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx_data_bridge config mapping - -{mapping, "emqx_data_bridge.bridges", "emqx_data_bridge.bridges", [ - {default, []}, - {datatype, string} -]}. - -% fields("emqx_data_bridge") -> -% [ -% {bridges, -% [fun(mapping) -> "emqx_data_bridge.bridges"; -% (type) -> list(); -% (_) -> undefined -% end]} -% ] \ No newline at end of file diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_data_bridge/rebar.config index cf4cfcf1b..2778ec974 100644 --- a/apps/emqx_data_bridge/rebar.config +++ b/apps/emqx_data_bridge/rebar.config @@ -1,5 +1,6 @@ {erl_opts, [debug_info]}. {deps, []}. +% {extra_src_dirs, [{"src", [{recursive, true}]}]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_connector/src/emqx_connector_ldap.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_ldap.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl index 3b0af5aa7..322bce1ef 100644 --- a/apps/emqx_connector/src/emqx_connector_ldap.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_ldap). --include("emqx_connector.hrl"). +-include("emqx_data_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_mongo.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_mongo.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl index 26a36dd0a..9e22e5ac5 100644 --- a/apps/emqx_connector/src/emqx_connector_mongo.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_mongo). --include("emqx_connector.hrl"). +-include("emqx_data_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_mysql.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl similarity index 100% rename from apps/emqx_connector/src/emqx_connector_mysql.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl similarity index 100% rename from apps/emqx_connector/src/emqx_connector_pgsql.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl diff --git a/apps/emqx_connector/src/emqx_connector_redis.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_redis.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl index 3eebbcf87..1037afede 100644 --- a/apps/emqx_connector/src/emqx_connector_redis.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_redis). --include("emqx_connector.hrl"). +-include("emqx_data_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_connector/src/emqx_connector_schema_lib.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl similarity index 99% rename from apps/emqx_connector/src/emqx_connector_schema_lib.erl rename to apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl index 03572d91d..aaacd0e80 100644 --- a/apps/emqx_connector/src/emqx_connector_schema_lib.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_schema_lib). --include("emqx_connector.hrl"). +-include("emqx_data_bridge.hrl"). -include_lib("typerefl/include/types.hrl"). -export([ relational_db_fields/0 diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src index 360511d9b..edfcd4a9d 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src @@ -5,7 +5,8 @@ {mod, {emqx_data_bridge_app, []}}, {applications, [kernel, - stdlib + stdlib, + ecpool ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index 5877cd3dd..54a228763 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -25,8 +25,9 @@ ]). load_bridges() -> - Bridges = proplists:get_value(bridges, - application:get_all_env(emqx_data_bridge), []), + ConfFile = filename:join([emqx:get_env(plugins_etc_dir), ?MODULE]) ++ ".conf", + {ok, #{<<"emqx_data_bridge">> := RawConfig}} = hocon:load(ConfFile), + Bridges = maps:get(<<"bridges">>, RawConfig, []), emqx_data_bridge_monitor:ensure_all_started(Bridges). resource_type(<<"mysql">>) -> emqx_connector_mysql; diff --git a/apps/emqx_resource/priv/emqx_resource.schema b/apps/emqx_resource/priv/emqx_resource.schema deleted file mode 100644 index 8246dc6a7..000000000 --- a/apps/emqx_resource/priv/emqx_resource.schema +++ /dev/null @@ -1,2 +0,0 @@ -%%-*- mode: erlang -*- -%% emqx-resource config mapping diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index 5ac46e0e3..d0dac7fe1 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -5,7 +5,4 @@ {emqx_retainer, {{enable_plugin_emqx_retainer}}}. {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. -{emqx_resource, {{enable_plugin_emqx_resource}}}. -{emqx_connector, {{enable_plugin_emqx_connector}}}. -{emqx_data_bridge, {{enable_plugin_emqx_data_bridge}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/rebar.config.erl b/rebar.config.erl index 0248e6dab..45baf0ea1 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -186,9 +186,6 @@ overlay_vars_rel(RelType) -> end, [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} - , {enable_plugin_emqx_resource, true} - , {enable_plugin_emqx_connector, true} - , {enable_plugin_emqx_data_bridge, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} @@ -247,6 +244,8 @@ relx_apps(ReleaseType) -> , {emqx_plugin_libs, load} , observer_cli , emqx_http_lib + , emqx_resource + , emqx_data_bridge ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] @@ -284,10 +283,8 @@ relx_plugin_apps(ReleaseType) -> , emqx_auth_mnesia , emqx_web_hook , emqx_recon - , emqx_resource - , emqx_connector - , emqx_data_bridge , emqx_rule_engine + , emqx_data_bridge , emqx_sasl ] ++ [emqx_telemetry || not is_enterprise()] From d9d5bc4fae37ce1f2e81981276aa36f87cc909be Mon Sep 17 00:00:00 2001 From: zhanghongtong Date: Thu, 17 Jun 2021 18:05:43 +0800 Subject: [PATCH 02/44] feat(connector): mysql and pgsql query support params --- .../src/connector/emqx_connector_mysql.erl | 4 +++- .../src/connector/emqx_connector_pgsql.erl | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl index e90d5f171..61e13901e 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl @@ -71,8 +71,10 @@ on_stop(InstId, #{poolname := PoolName}) -> emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> + on_query(InstId, {sql, SQL, []}, AfterQuery, #{poolname := PoolName} = State); +on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> logger:debug("mysql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]), - case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL]}, no_handover) of + case Result = ecpool:pick_and_do(PoolName, {mysql, query, [SQL, Params]}, no_handover) of {error, Reason} -> logger:debug("mysql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), emqx_resource:query_failed(AfterQuery); diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl b/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl index 329138ab1..0ff0d6779 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl +++ b/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl @@ -31,7 +31,7 @@ -export([connect/1]). --export([query/2]). +-export([query/3]). -export([do_health_check/1]). @@ -74,8 +74,10 @@ on_stop(InstId, #{poolname := PoolName}) -> emqx_plugin_libs_pool:stop_pool(PoolName). on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := PoolName} = State) -> + on_query(InstId, {sql, SQL, []}, AfterQuery, #{poolname := PoolName} = State); +on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) -> logger:debug("postgresql connector ~p received sql query: ~p, at state: ~p", [InstId, SQL, State]), - case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL]}, no_handover) of + case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of {error, Reason} -> logger:debug("postgresql connector ~p do sql query failed, sql: ~p, reason: ~p", [InstId, SQL, Reason]), emqx_resource:query_failed(AfterQuery); @@ -100,8 +102,8 @@ connect(Opts) -> Password = proplists:get_value(password, Opts), epgsql:connect(Host, Username, Password, conn_opts(Opts)). -query(Conn, SQL) -> - epgsql:squery(Conn, SQL). +query(Conn, SQL, Params) -> + epgsql:equery(Conn, SQL, Params). conn_opts(Opts) -> conn_opts(Opts, []). From 54dedc8343c4e2fae5a83498fe1f598b2642f327 Mon Sep 17 00:00:00 2001 From: turtleDeng Date: Thu, 17 Jun 2021 19:10:38 +0800 Subject: [PATCH 03/44] Revert "chore: mv emqx_connector to emqx_data_bridge" This reverts commit d640e2ccfa68a1c0529ef302b3eadc627d07f5e5. --- apps/emqx_connector/.gitignore | 19 ++++++++++ apps/emqx_connector/README.md | 27 ++++++++++++++ apps/emqx_connector/etc/emqx_connector.conf | 4 +++ .../include/emqx_connector.hrl} | 0 .../emqx_connector/priv/emqx_connector.schema | 2 ++ apps/emqx_connector/rebar.config | 13 +++++++ .../emqx_connector/src/emqx_connector.app.src | 17 +++++++++ apps/emqx_connector/src/emqx_connector.erl | 16 +++++++++ .../emqx_connector/src/emqx_connector_app.erl | 31 ++++++++++++++++ .../src}/emqx_connector_ldap.erl | 2 +- .../src}/emqx_connector_mongo.erl | 2 +- .../src}/emqx_connector_mysql.erl | 0 .../src}/emqx_connector_pgsql.erl | 0 .../src}/emqx_connector_redis.erl | 2 +- .../src}/emqx_connector_schema_lib.erl | 2 +- .../emqx_connector/src/emqx_connector_sup.erl | 36 +++++++++++++++++++ .../priv/emqx_data_bridge.schema | 16 +++++++++ apps/emqx_data_bridge/rebar.config | 1 - .../src/emqx_data_bridge.app.src | 3 +- .../emqx_data_bridge/src/emqx_data_bridge.erl | 5 ++- apps/emqx_resource/priv/emqx_resource.schema | 2 ++ data/loaded_plugins.tmpl | 3 ++ rebar.config.erl | 9 +++-- 23 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 apps/emqx_connector/.gitignore create mode 100644 apps/emqx_connector/README.md create mode 100644 apps/emqx_connector/etc/emqx_connector.conf rename apps/{emqx_data_bridge/include/emqx_data_bridge.hrl => emqx_connector/include/emqx_connector.hrl} (100%) create mode 100644 apps/emqx_connector/priv/emqx_connector.schema create mode 100644 apps/emqx_connector/rebar.config create mode 100644 apps/emqx_connector/src/emqx_connector.app.src create mode 100644 apps/emqx_connector/src/emqx_connector.erl create mode 100644 apps/emqx_connector/src/emqx_connector_app.erl rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_ldap.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_mongo.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_mysql.erl (100%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_pgsql.erl (100%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_redis.erl (99%) rename apps/{emqx_data_bridge/src/connector => emqx_connector/src}/emqx_connector_schema_lib.erl (99%) create mode 100644 apps/emqx_connector/src/emqx_connector_sup.erl create mode 100644 apps/emqx_data_bridge/priv/emqx_data_bridge.schema create mode 100644 apps/emqx_resource/priv/emqx_resource.schema diff --git a/apps/emqx_connector/.gitignore b/apps/emqx_connector/.gitignore new file mode 100644 index 000000000..f1c455451 --- /dev/null +++ b/apps/emqx_connector/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin +log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_connector/README.md b/apps/emqx_connector/README.md new file mode 100644 index 000000000..879669f93 --- /dev/null +++ b/apps/emqx_connector/README.md @@ -0,0 +1,27 @@ +# emqx_connector + +This application is a collection of `connectors`. + +A `connector` is a callback module of `emqx_resource` that maintains the data related to +external resources. Put all resource related callback modules in a single application is good as +we can put some util functions/modules here for reusing purpose. + +For example, a mysql connector is an emqx resource that maintains all the mysql connection +related parameters (configs) and the TCP connections to the mysql server. + +An mysql connector can be used as following: + +``` +(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose(). +[#{config => + #{auto_reconnect => true,cacertfile => [],certfile => [], + database => "mqtt",keyfile => [],password => "public", + pool_size => 1, + server => {{127,0,0,1},3306}, + ssl => false,user => "root",verify => false}, + id => <<"mysql-abc">>,mod => emqx_connector_mysql, + state => #{poolname => 'mysql-abc'}, + status => started}] +(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}). +{ok,[<<"count(1)">>],[[1]]} +``` diff --git a/apps/emqx_connector/etc/emqx_connector.conf b/apps/emqx_connector/etc/emqx_connector.conf new file mode 100644 index 000000000..db4402d47 --- /dev/null +++ b/apps/emqx_connector/etc/emqx_connector.conf @@ -0,0 +1,4 @@ +##-------------------------------------------------------------------- +## EMQ X CONNECTOR Plugin +##-------------------------------------------------------------------- + diff --git a/apps/emqx_data_bridge/include/emqx_data_bridge.hrl b/apps/emqx_connector/include/emqx_connector.hrl similarity index 100% rename from apps/emqx_data_bridge/include/emqx_data_bridge.hrl rename to apps/emqx_connector/include/emqx_connector.hrl diff --git a/apps/emqx_connector/priv/emqx_connector.schema b/apps/emqx_connector/priv/emqx_connector.schema new file mode 100644 index 000000000..b8476c4d9 --- /dev/null +++ b/apps/emqx_connector/priv/emqx_connector.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx_connector config mapping diff --git a/apps/emqx_connector/rebar.config b/apps/emqx_connector/rebar.config new file mode 100644 index 000000000..53b5c63e8 --- /dev/null +++ b/apps/emqx_connector/rebar.config @@ -0,0 +1,13 @@ +{erl_opts, [ + nowarn_unused_import, + debug_info +]}. + +{deps, [ + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}} +]}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx_connector]} +]}. diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src new file mode 100644 index 000000000..a821b8f13 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -0,0 +1,17 @@ +{application, emqx_connector, + [{description, "An OTP application"}, + {vsn, "0.1.0"}, + {registered, []}, + {mod, {emqx_connector_app, []}}, + {applications, + [kernel, + stdlib, + emqx_resource, + ecpool + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache 2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_connector/src/emqx_connector.erl b/apps/emqx_connector/src/emqx_connector.erl new file mode 100644 index 000000000..dd0359348 --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector.erl @@ -0,0 +1,16 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector). diff --git a/apps/emqx_connector/src/emqx_connector_app.erl b/apps/emqx_connector/src/emqx_connector_app.erl new file mode 100644 index 000000000..4bbad75cf --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_app.erl @@ -0,0 +1,31 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_connector_app). + +-behaviour(application). + +-emqx_plugin(?MODULE). + +-export([start/2, stop/1]). + +start(_StartType, _StartArgs) -> + emqx_connector_sup:start_link(). + +stop(_State) -> + ok. + +%% internal functions diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl b/apps/emqx_connector/src/emqx_connector_ldap.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl rename to apps/emqx_connector/src/emqx_connector_ldap.erl index 322bce1ef..3b0af5aa7 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_ldap.erl +++ b/apps/emqx_connector/src/emqx_connector_ldap.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_ldap). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl b/apps/emqx_connector/src/emqx_connector_mongo.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl rename to apps/emqx_connector/src/emqx_connector_mongo.erl index 9e22e5ac5..26a36dd0a 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_mongo.erl +++ b/apps/emqx_connector/src/emqx_connector_mongo.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_mongo). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl b/apps/emqx_connector/src/emqx_connector_mysql.erl similarity index 100% rename from apps/emqx_data_bridge/src/connector/emqx_connector_mysql.erl rename to apps/emqx_connector/src/emqx_connector_mysql.erl diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl similarity index 100% rename from apps/emqx_data_bridge/src/connector/emqx_connector_pgsql.erl rename to apps/emqx_connector/src/emqx_connector_pgsql.erl diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl b/apps/emqx_connector/src/emqx_connector_redis.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl rename to apps/emqx_connector/src/emqx_connector_redis.erl index 1037afede..3eebbcf87 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_redis.erl +++ b/apps/emqx_connector/src/emqx_connector_redis.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_redis). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx_resource/include/emqx_resource_behaviour.hrl"). diff --git a/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl b/apps/emqx_connector/src/emqx_connector_schema_lib.erl similarity index 99% rename from apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl rename to apps/emqx_connector/src/emqx_connector_schema_lib.erl index aaacd0e80..03572d91d 100644 --- a/apps/emqx_data_bridge/src/connector/emqx_connector_schema_lib.erl +++ b/apps/emqx_connector/src/emqx_connector_schema_lib.erl @@ -15,7 +15,7 @@ %%-------------------------------------------------------------------- -module(emqx_connector_schema_lib). --include("emqx_data_bridge.hrl"). +-include("emqx_connector.hrl"). -include_lib("typerefl/include/types.hrl"). -export([ relational_db_fields/0 diff --git a/apps/emqx_connector/src/emqx_connector_sup.erl b/apps/emqx_connector/src/emqx_connector_sup.erl new file mode 100644 index 000000000..603b9a8ad --- /dev/null +++ b/apps/emqx_connector/src/emqx_connector_sup.erl @@ -0,0 +1,36 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- +-module(emqx_connector_sup). + +-behaviour(supervisor). + +-export([start_link/0]). + +-export([init/1]). + +-define(SERVER, ?MODULE). + +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). + +init([]) -> + SupFlags = #{strategy => one_for_all, + intensity => 0, + period => 1}, + ChildSpecs = [], + {ok, {SupFlags, ChildSpecs}}. + +%% internal functions diff --git a/apps/emqx_data_bridge/priv/emqx_data_bridge.schema b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema new file mode 100644 index 000000000..c9cb3f2c0 --- /dev/null +++ b/apps/emqx_data_bridge/priv/emqx_data_bridge.schema @@ -0,0 +1,16 @@ +%%-*- mode: erlang -*- +%% emqx_data_bridge config mapping + +{mapping, "emqx_data_bridge.bridges", "emqx_data_bridge.bridges", [ + {default, []}, + {datatype, string} +]}. + +% fields("emqx_data_bridge") -> +% [ +% {bridges, +% [fun(mapping) -> "emqx_data_bridge.bridges"; +% (type) -> list(); +% (_) -> undefined +% end]} +% ] \ No newline at end of file diff --git a/apps/emqx_data_bridge/rebar.config b/apps/emqx_data_bridge/rebar.config index 2778ec974..cf4cfcf1b 100644 --- a/apps/emqx_data_bridge/rebar.config +++ b/apps/emqx_data_bridge/rebar.config @@ -1,6 +1,5 @@ {erl_opts, [debug_info]}. {deps, []}. -% {extra_src_dirs, [{"src", [{recursive, true}]}]}. {shell, [ % {config, "config/sys.config"}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src index edfcd4a9d..360511d9b 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.app.src +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.app.src @@ -5,8 +5,7 @@ {mod, {emqx_data_bridge_app, []}}, {applications, [kernel, - stdlib, - ecpool + stdlib ]}, {env,[]}, {modules, []}, diff --git a/apps/emqx_data_bridge/src/emqx_data_bridge.erl b/apps/emqx_data_bridge/src/emqx_data_bridge.erl index 54a228763..5877cd3dd 100644 --- a/apps/emqx_data_bridge/src/emqx_data_bridge.erl +++ b/apps/emqx_data_bridge/src/emqx_data_bridge.erl @@ -25,9 +25,8 @@ ]). load_bridges() -> - ConfFile = filename:join([emqx:get_env(plugins_etc_dir), ?MODULE]) ++ ".conf", - {ok, #{<<"emqx_data_bridge">> := RawConfig}} = hocon:load(ConfFile), - Bridges = maps:get(<<"bridges">>, RawConfig, []), + Bridges = proplists:get_value(bridges, + application:get_all_env(emqx_data_bridge), []), emqx_data_bridge_monitor:ensure_all_started(Bridges). resource_type(<<"mysql">>) -> emqx_connector_mysql; diff --git a/apps/emqx_resource/priv/emqx_resource.schema b/apps/emqx_resource/priv/emqx_resource.schema new file mode 100644 index 000000000..8246dc6a7 --- /dev/null +++ b/apps/emqx_resource/priv/emqx_resource.schema @@ -0,0 +1,2 @@ +%%-*- mode: erlang -*- +%% emqx-resource config mapping diff --git a/data/loaded_plugins.tmpl b/data/loaded_plugins.tmpl index d0dac7fe1..5ac46e0e3 100644 --- a/data/loaded_plugins.tmpl +++ b/data/loaded_plugins.tmpl @@ -5,4 +5,7 @@ {emqx_retainer, {{enable_plugin_emqx_retainer}}}. {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}. {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}. +{emqx_resource, {{enable_plugin_emqx_resource}}}. +{emqx_connector, {{enable_plugin_emqx_connector}}}. +{emqx_data_bridge, {{enable_plugin_emqx_data_bridge}}}. {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}. diff --git a/rebar.config.erl b/rebar.config.erl index 45baf0ea1..0248e6dab 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -186,6 +186,9 @@ overlay_vars_rel(RelType) -> end, [ {enable_plugin_emqx_rule_engine, RelType =:= cloud} , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge} + , {enable_plugin_emqx_resource, true} + , {enable_plugin_emqx_connector, true} + , {enable_plugin_emqx_data_bridge, true} , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce , {enable_plugin_emqx_recon, true} , {enable_plugin_emqx_retainer, true} @@ -244,8 +247,6 @@ relx_apps(ReleaseType) -> , {emqx_plugin_libs, load} , observer_cli , emqx_http_lib - , emqx_resource - , emqx_data_bridge ] ++ [emqx_modules || not is_enterprise()] ++ [emqx_license || is_enterprise()] @@ -283,8 +284,10 @@ relx_plugin_apps(ReleaseType) -> , emqx_auth_mnesia , emqx_web_hook , emqx_recon - , emqx_rule_engine + , emqx_resource + , emqx_connector , emqx_data_bridge + , emqx_rule_engine , emqx_sasl ] ++ [emqx_telemetry || not is_enterprise()] From 1e0f6f4dea38e8649717e8a4f236e0002462f8b4 Mon Sep 17 00:00:00 2001 From: DDDHuang <904897578@qq.com> Date: Thu, 10 Jun 2021 16:59:27 +0800 Subject: [PATCH 04/44] chore: more node info --- .../src/emqx_mgmt_api_nodes.erl | 28 +++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl index 2be151aaa..c24e46de9 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_nodes.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_nodes.erl @@ -41,6 +41,30 @@ get(#{node := Node}, _Params) -> format(Node, {error, Reason}) -> #{node => Node, error => Reason}; format(_Node, Info = #{memory_total := Total, memory_used := Used}) -> - Info#{memory_total := emqx_mgmt_util:kmg(Total), - memory_used := emqx_mgmt_util:kmg(Used)}. + {ok, SysPathBinary} = file:get_cwd(), + SysPath = list_to_binary(SysPathBinary), + ConfigPath = <>, + LogPath = case log_path() of + undefined -> + <<"not found">>; + Path0 -> + Path = list_to_binary(Path0), + <> + end, + Info#{ memory_total := emqx_mgmt_util:kmg(Total) + , memory_used := emqx_mgmt_util:kmg(Used) + , sys_path => SysPath + , config_path => ConfigPath + , log_path => LogPath}. +log_path() -> + Configs = logger:get_handler_config(), + get_log_path(Configs). + +get_log_path([#{id := file} = LoggerConfig | _LoggerConfigs]) -> + Config = maps:get(config, LoggerConfig), + maps:get(file, Config); +get_log_path([_LoggerConfig | LoggerConfigs]) -> + get_log_path(LoggerConfigs); +get_log_path([]) -> + undefined. From 6865f36d908115303a28e55db4f5cb2ed1585d64 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 17 Jun 2021 16:35:03 +0200 Subject: [PATCH 05/44] chore(rebar.config): use hocon 0.6.0 hocon 0.6.0 added supports for sensitive config flagging sub-struct type checking, and remote struct referencing --- apps/emqx/rebar.config | 2 +- rebar.config | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index a33aa2e64..693e96c74 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -16,7 +16,7 @@ , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.1"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} %% todo delete when plugins use hocon - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.6.0"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}} , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} diff --git a/rebar.config b/rebar.config index 03a36570f..fa8752774 100644 --- a/rebar.config +++ b/rebar.config @@ -52,7 +52,7 @@ , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1 , {getopt, "1.0.1"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.1"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.6.0"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.2.1"}}} ]}. From 36cd03cc98d9ee605ce9722d878a90f9f334b845 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 17 Jun 2021 16:46:45 +0200 Subject: [PATCH 06/44] fix(emqx_schema): add 'sensitive' flag for passowrd configs --- apps/emqx/src/emqx_schema.erl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index bee052926..cb631ea22 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -113,7 +113,11 @@ fields("rlog") -> fields("node") -> [ {"name", t(string(), "vm_args.-name", "emqx@127.0.0.1", "EMQX_NODE_NAME")} , {"ssl_dist_optfile", t(string(), "vm_args.-ssl_dist_optfile", undefined)} - , {"cookie", t(string(), "vm_args.-setcookie", "emqxsecretcookie", "EMQX_NODE_COOKIE")} + , {"cookie", hoconsc:t(string(), #{mapping => "vm_args.-setcookie", + default => "emqxsecretcookie", + sensitive => true, + override_env => "EMQX_NODE_COOKIE" + })} , {"data_dir", t(string(), "emqx.data_dir", undefined)} , {"heartbeat", t(flag(), undefined, false)} , {"async_threads", t(range(1, 1024), "vm_args.+A", undefined)} @@ -1098,7 +1102,10 @@ ssl(Mapping, Defaults) -> , {"honor_cipher_order", t(flag(), M("honor_cipher_order"), D("honor_cipher_order"))} , {"handshake_timeout", t(duration(), M("handshake_timeout"), D("handshake_timeout"))} , {"depth", t(integer(), M("depth"), D("depth"))} - , {"password", t(string(), M("key_password"), D("key_password"))} + , {"password", hoconsc:t(string(), #{mapping => M("key_password"), + default => D("key_password"), + sensitive => true + })} , {"dhfile", t(string(), M("dhfile"), D("dhfile"))} , {"server_name_indication", t(union(disable, string()), M("server_name_indication"), D("server_name_indication"))} From 2919623563d61926b38a4ef176948d215ed94f32 Mon Sep 17 00:00:00 2001 From: Zaiming Shi Date: Thu, 17 Jun 2021 17:14:57 +0200 Subject: [PATCH 07/44] chore(bin/emqx): print env override when generating config --- bin/emqx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/emqx b/bin/emqx index 3e54dc17b..a5f76ac72 100755 --- a/bin/emqx +++ b/bin/emqx @@ -230,7 +230,7 @@ generate_config() { ## ths command populates two files: app.