From 9673a73411fe30615b44e5dec2c8ecd663044ff2 Mon Sep 17 00:00:00 2001 From: CrazyWisdom Date: Sun, 23 Apr 2023 09:08:48 +0800 Subject: [PATCH 01/92] docs(README): add mqtt broker link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 280371a41..f8aa76af2 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ [![YouTube](https://img.shields.io/badge/Subscribe-EMQ-FF0000?logo=youtube)](https://www.youtube.com/channel/UC5FjR77ErAxvZENEWzQaO5Q) -EMQX is the world's most scalable open-source MQTT broker with a high performance that connects 100M+ IoT devices in 1 cluster, while maintaining 1M message per second throughput and sub-millisecond latency. +EMQX is the world's most scalable open-source [MQTT broker](https://www.emqx.com/en/blog/the-ultimate-guide-to-mqtt-broker-comparison) with a high performance that connects 100M+ IoT devices in 1 cluster, while maintaining 1M message per second throughput and sub-millisecond latency. EMQX supports multiple open standard protocols like MQTT, HTTP, QUIC, and WebSocket. It’s 100% compliant with MQTT 5.0 and 3.x standard, and secures bi-directional communication with MQTT over TLS/SSL and various authentication mechanisms. From c216dfd96b7378a1609b3d9eff305110cf9292c9 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 30 Jun 2023 15:40:40 -0300 Subject: [PATCH 02/92] fix(mysql_bridge): make nxdomain a 400 API error Fixes https://emqx.atlassian.net/browse/EMQX-10460 --- apps/emqx_bridge/src/emqx_bridge_api.erl | 10 +++++++--- apps/emqx_mysql/rebar.config | 2 +- changes/ee/fix-11175.en.md | 1 + 3 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 changes/ee/fix-11175.en.md diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 7a03b24ca..57933029d 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -985,9 +985,13 @@ call_operation(NodeOrAll, OperFunc, Args = [_Nodes, BridgeType, BridgeName]) -> {error, timeout} -> ?SERVICE_UNAVAILABLE(<<"Request timeout">>); {error, {start_pool_failed, Name, Reason}} -> - ?SERVICE_UNAVAILABLE( - bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason])) - ); + Msg = bin(io_lib:format("Failed to start ~p pool for reason ~p", [Name, Reason])), + case Reason of + nxdomain -> + ?BAD_REQUEST(Msg); + _ -> + ?SERVICE_UNAVAILABLE(Msg) + end; {error, not_found} -> BridgeId = emqx_bridge_resource:bridge_id(BridgeType, BridgeName), ?SLOG(warning, #{ diff --git a/apps/emqx_mysql/rebar.config b/apps/emqx_mysql/rebar.config index 58b6665ad..668e437f3 100644 --- a/apps/emqx_mysql/rebar.config +++ b/apps/emqx_mysql/rebar.config @@ -3,7 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ %% NOTE: mind ecpool version when updating eredis_cluster version - {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}}, + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.3"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}} ]}. diff --git a/changes/ee/fix-11175.en.md b/changes/ee/fix-11175.en.md new file mode 100644 index 000000000..24a9def70 --- /dev/null +++ b/changes/ee/fix-11175.en.md @@ -0,0 +1 @@ +Now when using a nonexistent hostname for connecting to MySQL will result in a 400 error rather than 503 in the HTTP API. From 263890be47e80eaa5ee6dea541d6407d26537250 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Mon, 3 Jul 2023 16:07:16 +0200 Subject: [PATCH 03/92] fix: limit mqtt max_packet_size to 256MB --- apps/emqx/src/emqx_schema.erl | 6 +++++- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 2 ++ changes/ce/fix-11184.en.md | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 changes/ce/fix-11184.en.md diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index 8509dc245..f552fae7f 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -30,6 +30,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("logger.hrl"). +-define(MAX_INT_MQTT_PACKET_SIZE, 268435456). -define(MAX_INT_TIMEOUT_MS, 4294967295). %% floor(?MAX_INT_TIMEOUT_MS / 1000). -define(MAX_INT_TIMEOUT_S, 4294967). @@ -45,6 +46,7 @@ -type timeout_duration_s() :: 0..?MAX_INT_TIMEOUT_S. -type timeout_duration_ms() :: 0..?MAX_INT_TIMEOUT_MS. -type bytesize() :: integer(). +-type mqtt_max_packet_size() :: 1..?MAX_INT_MQTT_PACKET_SIZE. -type wordsize() :: bytesize(). -type percent() :: float(). -type file() :: string(). @@ -71,6 +73,7 @@ -typerefl_from_string({timeout_duration_s/0, emqx_schema, to_timeout_duration_s}). -typerefl_from_string({timeout_duration_ms/0, emqx_schema, to_timeout_duration_ms}). -typerefl_from_string({bytesize/0, emqx_schema, to_bytesize}). +-typerefl_from_string({mqtt_max_packet_size/0, emqx_schema, to_bytesize}). -typerefl_from_string({wordsize/0, emqx_schema, to_wordsize}). -typerefl_from_string({percent/0, emqx_schema, to_percent}). -typerefl_from_string({comma_separated_list/0, emqx_schema, to_comma_separated_list}). @@ -151,6 +154,7 @@ timeout_duration_s/0, timeout_duration_ms/0, bytesize/0, + mqtt_max_packet_size/0, wordsize/0, percent/0, file/0, @@ -3357,7 +3361,7 @@ mqtt_general() -> )}, {"max_packet_size", sc( - bytesize(), + mqtt_max_packet_size(), #{ default => <<"1MB">>, desc => ?DESC(mqtt_max_packet_size) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 47acee58b..c22a5318b 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -852,6 +852,8 @@ typename_to_spec("timeout()", _Mod) -> }; typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>}; +typename_to_spec("mqtt_max_packet_size()", _Mod) -> + #{type => string, example => <<"32MB">>}; typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>}; typename_to_spec("map()", _Mod) -> diff --git a/changes/ce/fix-11184.en.md b/changes/ce/fix-11184.en.md new file mode 100644 index 000000000..46a790a72 --- /dev/null +++ b/changes/ce/fix-11184.en.md @@ -0,0 +1 @@ +Config value for `max_packet_size` has a max value of 256MB defined by protocol. This is now enforced and any configuration with a value greater than that will break. From 0483c6afef0437e405787d379c59a21009d9bfd3 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 5 Jul 2023 14:13:25 +0800 Subject: [PATCH 04/92] fix(stomp): not allow to create same subscriptions --- .../emqx_gateway_stomp/src/emqx_gateway_stomp.app.src | 2 +- apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src index 1fda99700..bcc018ad4 100644 --- a/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src +++ b/apps/emqx_gateway_stomp/src/emqx_gateway_stomp.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_stomp, [ {description, "Stomp Gateway"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index eef30b3dd..8b7e165d5 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -695,12 +695,15 @@ check_subscribed_status( ) -> MountedTopic = emqx_mountpoint:mount(Mountpoint, ParsedTopic), case lists:keyfind(SubId, 1, Subs) of - {SubId, MountedTopic, _Ack, _} -> - ok; - {SubId, _OtherTopic, _Ack, _} -> + {SubId, _MountedTopic, _Ack, _} -> {error, subscription_id_inused}; false -> - ok + case lists:keyfind(MountedTopic, 2, Subs) of + {_OtherSubId, MountedTopic, _Ack, _} -> + {error, subscription_inused}; + false -> + ok + end end. check_sub_acl( From aaf015c06f0123f07dcd4194c580c84684929d44 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 5 Jul 2023 14:57:20 +0800 Subject: [PATCH 05/92] test(stomp): cover subscription in use --- .../src/emqx_stomp_channel.erl | 21 +- .../test/emqx_stomp_SUITE.erl | 182 ++++++++++-------- 2 files changed, 122 insertions(+), 81 deletions(-) diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index 8b7e165d5..3577c0a60 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -508,9 +508,13 @@ handle_in( handle_out_and_update(receipt, receipt_id(Headers), NChannel1) end; {error, subscription_id_inused, NChannel} -> - ErrMsg = io_lib:format("Subscription id ~w is in used", [SubId]), + ErrMsg = io_lib:format("Subscription id ~s is in used", [SubId]), ErrorFrame = error_frame(receipt_id(Headers), ErrMsg), shutdown(subscription_id_inused, ErrorFrame, NChannel); + {error, topic_already_subscribed, NChannel} -> + ErrMsg = io_lib:format("Topic ~s already in subscribed", [Topic]), + ErrorFrame = error_frame(receipt_id(Headers), ErrMsg), + shutdown(topic_already_subscribed, ErrorFrame, NChannel); {error, acl_denied, NChannel} -> ErrMsg = io_lib:format("Insufficient permissions for ~s", [Topic]), ErrorFrame = error_frame(receipt_id(Headers), ErrMsg), @@ -700,7 +704,7 @@ check_subscribed_status( false -> case lists:keyfind(MountedTopic, 2, Subs) of {_OtherSubId, MountedTopic, _Ack, _} -> - {error, subscription_inused}; + {error, topic_already_subscribed}; false -> ok end @@ -829,13 +833,20 @@ handle_call( NSubs = [{SubId, MountedTopic, <<"auto">>, NSubOpts} | Subs], NChannel1 = NChannel#channel{subscriptions = NSubs}, reply({ok, {MountedTopic, NSubOpts}}, [{event, updated}], NChannel1); - {error, ErrMsg, NChannel} -> + {error, ErrCode, NChannel} -> ?SLOG(error, #{ msg => "failed_to_subscribe_topic", topic => Topic, - reason => ErrMsg + reason => ErrCode }), - reply({error, ErrMsg}, NChannel) + ErrMsg = + case ErrCode of + subscription_id_inused -> + io_lib:format("Subscription id ~s is in used", [SubId]); + topic_already_subscribed -> + io_lib:format("Topic ~s already in subscribed", [Topic]) + end, + reply({error, lists:flatten(ErrMsg)}, NChannel) end end; handle_call( diff --git a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl index 47b191855..d9f0f4ce2 100644 --- a/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl +++ b/apps/emqx_gateway_stomp/test/emqx_stomp_SUITE.erl @@ -162,62 +162,28 @@ t_heartbeat(_) -> t_subscribe(_) -> with_connection(fun(Sock) -> - gen_tcp:send( - Sock, - serialize( - <<"CONNECT">>, - [ - {<<"accept-version">>, ?STOMP_VER}, - {<<"host">>, <<"127.0.0.1:61613">>}, - {<<"login">>, <<"guest">>}, - {<<"passcode">>, <<"guest">>}, - {<<"heart-beat">>, <<"0,0">>} - ] - ) - ), - {ok, Data} = gen_tcp:recv(Sock, 0), - {ok, - #stomp_frame{ - command = <<"CONNECTED">>, - headers = _, - body = _ - }, - _, _} = parse(Data), + ok = send_connection_frame(Sock, <<"guest">>, <<"guest">>), + ?assertMatch({ok, #stomp_frame{command = <<"CONNECTED">>}}, recv_a_frame(Sock)), - %% Subscribe - gen_tcp:send( - Sock, - serialize( - <<"SUBSCRIBE">>, - [ - {<<"id">>, 0}, - {<<"destination">>, <<"/queue/foo">>}, - {<<"ack">>, <<"auto">>} - ] - ) - ), + ok = send_subscribe_frame(Sock, 0, <<"/queue/foo">>), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), %% 'user-defined' header will be retain - gen_tcp:send( - Sock, - serialize( - <<"SEND">>, - [ - {<<"destination">>, <<"/queue/foo">>}, - {<<"user-defined">>, <<"emq">>} - ], - <<"hello">> - ) - ), + ok = send_message_frame(Sock, <<"/queue/foo">>, <<"hello">>, [ + {<<"user-defined">>, <<"emq">>} + ]), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), - {ok, Data1} = gen_tcp:recv(Sock, 0, 1000), - {ok, - Frame = #stomp_frame{ + {ok, Frame} = recv_a_frame(Sock), + + ?assertMatch( + #stomp_frame{ command = <<"MESSAGE">>, headers = _, body = <<"hello">> }, - _, _} = parse(Data1), + Frame + ), lists:foreach( fun({Key, Val}) -> Val = proplists:get_value(Key, Frame#stomp_frame.headers) @@ -234,43 +200,80 @@ t_subscribe(_) -> ?assertMatch(#{subscriptions_cnt := 1}, ClientInfo1), %% Unsubscribe - gen_tcp:send( - Sock, - serialize( - <<"UNSUBSCRIBE">>, - [ - {<<"id">>, 0}, - {<<"receipt">>, <<"12345">>} - ] - ) - ), - - {ok, Data2} = gen_tcp:recv(Sock, 0, 1000), - - {ok, - #stomp_frame{ - command = <<"RECEIPT">>, - headers = [{<<"receipt-id">>, <<"12345">>}], - body = _ - }, - _, _} = parse(Data2), + ok = send_unsubscribe_frame(Sock, 0), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), %% assert subscription stats [ClientInfo2] = clients(), ?assertMatch(#{subscriptions_cnt := 0}, ClientInfo2), - gen_tcp:send( - Sock, - serialize( - <<"SEND">>, - [{<<"destination">>, <<"/queue/foo">>}], - <<"You will not receive this msg">> - ) - ), + ok = send_message_frame(Sock, <<"/queue/foo">>, <<"You will not receive this msg">>), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)), {error, timeout} = gen_tcp:recv(Sock, 0, 500) end). +t_subscribe_inuse(_) -> + UsedTopic = <<"/queue/foo">>, + UsedSubId = <<"0">>, + Setup = + fun(Sock) -> + ok = send_connection_frame(Sock, <<"guest">>, <<"guest">>), + ?assertMatch({ok, #stomp_frame{command = <<"CONNECTED">>}}, recv_a_frame(Sock)), + ok = send_subscribe_frame(Sock, UsedSubId, UsedTopic), + ?assertMatch({ok, #stomp_frame{command = <<"RECEIPT">>}}, recv_a_frame(Sock)) + end, + TopicIdInuse = + fun(Sock) -> + Setup(Sock), + %% topic-id is in use + ok = send_subscribe_frame(Sock, UsedSubId, <<"/queue/bar">>), + + {ok, ErrorFrame} = recv_a_frame(Sock), + ?assertMatch(#stomp_frame{command = <<"ERROR">>}, ErrorFrame), + ?assertEqual(<<"Subscription id 0 is in used">>, ErrorFrame#stomp_frame.body), + ?assertMatch({error, closed}, gen_tcp:recv(Sock, 0)) + end, + + SubscriptionInuse = + fun(Sock) -> + Setup(Sock), + %% topic is in use + ok = send_subscribe_frame(Sock, 1, UsedTopic), + + {ok, ErrorFrame} = recv_a_frame(Sock), + ?assertMatch(#stomp_frame{command = <<"ERROR">>}, ErrorFrame), + ?assertEqual(<<"Topic /queue/foo already in subscribed">>, ErrorFrame#stomp_frame.body), + ?assertMatch({error, closed}, gen_tcp:recv(Sock, 0)) + end, + + TopicIdInuseViaHttp = + fun(Sock) -> + Setup(Sock), + %% assert subscription stats + [#{clientid := ClientId}] = clients(), + {error, ErrMsg} = create_subscription(ClientId, <<"/queue/bar">>, UsedSubId), + ?assertEqual(<<"Subscription id 0 is in used">>, ErrMsg), + + ok = send_disconnect_frame(Sock) + end, + + SubscriptionInuseViaHttp = + fun(Sock) -> + Setup(Sock), + %% assert subscription stats + [#{clientid := ClientId}] = clients(), + {error, ErrMsg} = create_subscription(ClientId, UsedTopic, <<"1">>), + ?assertEqual(<<"Topic /queue/foo already in subscribed">>, ErrMsg), + + ok = send_disconnect_frame(Sock) + end, + + with_connection(TopicIdInuse), + with_connection(SubscriptionInuse), + with_connection(TopicIdInuseViaHttp), + with_connection(SubscriptionInuseViaHttp). + t_transaction(_) -> with_connection(fun(Sock) -> gen_tcp:send( @@ -1072,6 +1075,7 @@ recv_a_frame(Sock) -> {ok, Frame, Rest, NParser} -> put(parser, NParser), put(rest, Rest), + ct:pal("recv_a_frame: ~p~n", [Frame]), {ok, Frame}; {error, _} = Err -> erase(parser), @@ -1124,11 +1128,23 @@ send_subscribe_frame(Sock, Id, Topic) -> ], ok = gen_tcp:send(Sock, serialize(<<"SUBSCRIBE">>, Headers)). +send_unsubscribe_frame(Sock, Id) when is_integer(Id) -> + Headers = + [ + {<<"id">>, Id}, + {<<"receipt">>, <<"rp-", (integer_to_binary(Id))/binary>>} + ], + gen_tcp:send(Sock, serialize(<<"UNSUBSCRIBE">>, Headers)). + send_message_frame(Sock, Topic, Payload) -> + send_message_frame(Sock, Topic, Payload, []). + +send_message_frame(Sock, Topic, Payload, Headers0) -> Headers = [ {<<"destination">>, Topic}, {<<"receipt">>, <<"rp-", Topic/binary>>} + | Headers0 ], ok = gen_tcp:send(Sock, serialize(<<"SEND">>, Headers, Payload)). @@ -1142,3 +1158,17 @@ send_disconnect_frame(Sock, ReceiptId) -> clients() -> {200, Clients} = request(get, "/gateways/stomp/clients"), maps:get(data, Clients). + +create_subscription(ClientId, Topic, SubId) -> + Path = io_lib:format("/gateways/stomp/clients/~s/subscriptions", [ClientId]), + Body = #{ + topic => Topic, + qos => 1, + sub_props => #{subid => SubId} + }, + case request(post, Path, Body) of + {201, _} -> + ok; + {400, #{message := Message}} -> + {error, Message} + end. From 9fed5233bd30ffa628b96ec3a271cfb90f60a8e0 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 5 Jul 2023 15:26:25 +0800 Subject: [PATCH 06/92] chore: update changes --- changes/ce/fix-11195.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-11195.en.md diff --git a/changes/ce/fix-11195.en.md b/changes/ce/fix-11195.en.md new file mode 100644 index 000000000..4c2d8b6b7 --- /dev/null +++ b/changes/ce/fix-11195.en.md @@ -0,0 +1 @@ +Avoid to create duplicated subscription by HTTP API or client in Stomp gateway From dca6fe62006ee7ad01ecddbb0c3f7e8ac02a4e68 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Jul 2023 12:48:09 +0300 Subject: [PATCH 07/92] fix(rebalance): fix global status evaluation on replicant nodes --- .../src/emqx_node_rebalance_status.erl | 2 +- .../test/emqx_node_rebalance_status_SUITE.erl | 65 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl index 1d45d64e8..a0102c4f4 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_status.erl @@ -51,7 +51,7 @@ format_local_status(Status) -> -spec global_status() -> #{rebalances := [{node(), map()}], evacuations := [{node(), map()}]}. global_status() -> - Nodes = mria_mnesia:running_nodes(), + Nodes = emqx:running_nodes(), {RebalanceResults, _} = emqx_node_rebalance_status_proto_v1:rebalance_status(Nodes), Rebalances = [ {Node, coordinator_rebalance(Status)} diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl new file mode 100644 index 000000000..167c37d8c --- /dev/null +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_status_SUITE.erl @@ -0,0 +1,65 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_node_rebalance_status_SUITE). + +-compile(export_all). +-compile(nowarn_export_all). + +-include_lib("common_test/include/ct.hrl"). +-include_lib("stdlib/include/assert.hrl"). + +all() -> emqx_common_test_helpers:all(?MODULE). + +suite() -> + [{timetrap, {seconds, 90}}]. + +init_per_suite(Config) -> + WorkDir = ?config(priv_dir, Config), + Apps = [ + emqx_conf, + emqx, + emqx_node_rebalance + ], + Cluster = [ + {emqx_node_rebalance_status_SUITE1, #{ + role => core, + apps => Apps + }}, + {emqx_node_rebalance_status_SUITE2, #{ + role => replicant, + apps => Apps + }} + ], + Nodes = emqx_cth_cluster:start(Cluster, #{work_dir => WorkDir}), + [{cluster_nodes, Nodes} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)), + ok. + +%%-------------------------------------------------------------------- +%% Tests +%%-------------------------------------------------------------------- + +t_cluster_status(Config) -> + [CoreNode, ReplicantNode] = ?config(cluster_nodes, Config), + ok = emqx_node_rebalance_api_proto_v1:node_rebalance_evacuation_start(CoreNode, #{}), + + ?assertMatch( + #{evacuations := [_], rebalances := []}, + rpc:call(ReplicantNode, emqx_node_rebalance_status, global_status, []) + ). From f174cb656c28a1c790448193b5c04de59dec6666 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Jul 2023 15:38:41 +0300 Subject: [PATCH 08/92] fix(rebalance): fix swagger examples in api-doc --- .../src/emqx_dashboard_swagger.erl | 46 ++++++++++--------- .../test/emqx_swagger_requestBody_SUITE.erl | 18 ++++++++ .../src/emqx_node_rebalance_api.erl | 35 ++++++++------ 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 47acee58b..f829c8b08 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -897,15 +897,25 @@ typename_to_spec("json_binary()", _Mod) -> typename_to_spec("port_number()", _Mod) -> range("1..65535"); typename_to_spec(Name, Mod) -> - Spec = range(Name), - Spec1 = remote_module_type(Spec, Name, Mod), - Spec2 = typerefl_array(Spec1, Name, Mod), - Spec3 = integer(Spec2, Name), - Spec3 =:= nomatch andalso - throw({error, #{msg => <<"Unsupported Type">>, type => Name, module => Mod}}), - Spec3. + try_convert_to_spec(Name, Mod, [ + fun try_remote_module_type/2, + fun try_typerefl_array/2, + fun try_range/2, + fun try_integer/2 + ]). range(Name) -> + #{} = try_range(Name, undefined). + +try_convert_to_spec(Name, Mod, []) -> + throw({error, #{msg => <<"Unsupported Type">>, type => Name, module => Mod}}); +try_convert_to_spec(Name, Mod, [Converter | Rest]) -> + case Converter(Name, Mod) of + nomatch -> try_convert_to_spec(Name, Mod, Rest); + Spec -> Spec + end. + +try_range(Name, _Mod) -> case string:split(Name, "..") of %% 1..10 1..inf -inf..10 [MinStr, MaxStr] -> @@ -917,39 +927,33 @@ range(Name) -> end. %% Module:Type -remote_module_type(nomatch, Name, Mod) -> +try_remote_module_type(Name, Mod) -> case string:split(Name, ":") of [_Module, Type] -> typename_to_spec(Type, Mod); _ -> nomatch - end; -remote_module_type(Spec, _Name, _Mod) -> - Spec. + end. -%% [string()] or [integer()] or [xxx]. -typerefl_array(nomatch, Name, Mod) -> +%% [string()] or [integer()] or [xxx] or [xxx,...] +try_typerefl_array(Name, Mod) -> case string:trim(Name, leading, "[") of Name -> nomatch; Name1 -> - case string:trim(Name1, trailing, "]") of + case string:trim(Name1, trailing, ",.]") of Name1 -> notmatch; Name2 -> Schema = typename_to_spec(Name2, Mod), #{type => array, items => Schema} end - end; -typerefl_array(Spec, _Name, _Mod) -> - Spec. + end. %% integer(1) -integer(nomatch, Name) -> +try_integer(Name, _Mod) -> case string:to_integer(Name) of {Int, []} -> #{type => integer, enum => [Int], default => Int}; _ -> nomatch - end; -integer(Spec, _Name) -> - Spec. + end. add_integer_prop(Schema, Key, Value) -> case string:to_integer(Value) of diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index e8c79c57c..4bc0f4a7c 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -105,6 +105,20 @@ t_deprecated(_Config) -> emqx_dashboard_swagger:components([{?MODULE, deprecated_ref}], #{}) ). +t_nonempty_list(_Config) -> + ?assertMatch( + [ + #{ + <<"emqx_swagger_requestBody_SUITE.nonempty_list_ref">> := + #{ + <<"properties">> := + [{<<"list">>, #{items := #{type := string}, type := array}}] + } + } + ], + emqx_dashboard_swagger:components([{?MODULE, nonempty_list_ref}], #{}) + ). + t_nest_object(_Config) -> GoodRef = <<"#/components/schemas/emqx_swagger_requestBody_SUITE.good_ref">>, Spec = #{ @@ -829,6 +843,10 @@ fields(deprecated_ref) -> {tag1, mk(binary(), #{desc => <<"tag1">>, deprecated => {since, "4.3.0"}})}, {tag2, mk(binary(), #{desc => <<"tag2">>, deprecated => true})}, {tag3, mk(binary(), #{desc => <<"tag3">>, deprecated => false})} + ]; +fields(nonempty_list_ref) -> + [ + {list, mk(nonempty_list(binary()), #{})} ]. enable(type) -> boolean(); diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl index 1d25bfb33..fb27c0a30 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance_api.erl @@ -709,25 +709,30 @@ fields(global_status) -> rebalance_example() -> #{ - wait_health_check => <<"10s">>, - conn_evict_rate => 10, - sess_evict_rate => 20, - abs_conn_threshold => 10, - rel_conn_threshold => 1.5, - abs_sess_threshold => 10, - rel_sess_threshold => 1.5, - wait_takeover => <<"10s">>, - nodes => [<<"othernode@127.0.0.1">>] + rebalance => + #{ + wait_health_check => <<"10s">>, + conn_evict_rate => 10, + sess_evict_rate => 20, + abs_conn_threshold => 10, + rel_conn_threshold => 1.5, + abs_sess_threshold => 10, + rel_sess_threshold => 1.5, + wait_takeover => <<"10s">>, + nodes => [<<"othernode@127.0.0.1">>] + } }. rebalance_evacuation_example() -> #{ - wait_health_check => <<"10s">>, - conn_evict_rate => 100, - sess_evict_rate => 100, - redirect_to => <<"othernode:1883">>, - wait_takeover => <<"10s">>, - migrate_to => [<<"othernode@127.0.0.1">>] + evacuation => #{ + wait_health_check => <<"10s">>, + conn_evict_rate => 100, + sess_evict_rate => 100, + redirect_to => <<"othernode:1883">>, + wait_takeover => <<"10s">>, + migrate_to => [<<"othernode@127.0.0.1">>] + } }. local_status_response_schema() -> From 9af0030538bf776c94fe1970787c9722d856b75a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Jul 2023 18:39:01 +0300 Subject: [PATCH 09/92] chore: bump to v5.1.1 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index c83444efc..57a39816d 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -32,7 +32,7 @@ %% `apps/emqx/src/bpapi/README.md' %% Opensource edition --define(EMQX_RELEASE_CE, "5.1.0"). +-define(EMQX_RELEASE_CE, "5.1.1"). %% Enterprise edition -define(EMQX_RELEASE_EE, "5.1.0"). diff --git a/deploy/charts/emqx/Chart.yaml b/deploy/charts/emqx/Chart.yaml index 045001b79..a2262da8b 100644 --- a/deploy/charts/emqx/Chart.yaml +++ b/deploy/charts/emqx/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.1.0 +version: 5.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.1.0 +appVersion: 5.1.1 From 659980f69d1d4529ab79970ad9b345c898597269 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Jul 2023 18:48:33 +0300 Subject: [PATCH 10/92] docs: Generate changelog for v5.1.1 --- changes/v5.1.1.en.md | 104 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 changes/v5.1.1.en.md diff --git a/changes/v5.1.1.en.md b/changes/v5.1.1.en.md new file mode 100644 index 000000000..521a96af8 --- /dev/null +++ b/changes/v5.1.1.en.md @@ -0,0 +1,104 @@ +# v5.1.1 + +## Enhancements + +- [#10667](https://github.com/emqx/emqx/pull/10667) The MongoDB connector and bridge have been refactored to a separate app to improve code structure. + +- [#11115](https://github.com/emqx/emqx/pull/11115) Added info logs to indicate when buffered messages are dropped due to time-to-live (TTL) expiration. + +- [#11133](https://github.com/emqx/emqx/pull/11133) Rename `deliver_rate` to `delivery_rate` in the configuration of `retainer`. + +- [#11137](https://github.com/emqx/emqx/pull/11137) Refactors the dashboard listener configuration to use a nested `ssl_options` field for ssl settings. + +- [#11138](https://github.com/emqx/emqx/pull/11138) - Change k8s `api_server` default value from `http://127.0.0.1:9091` to `https://kubernetes.default.svc:443` + - `emqx_ctl conf show cluster` no longer displays irrelevant configuration items, such as when `discovery_strategy=static`, + it will not display configuration information related to `etcd/k8s/dns`. + - Remove `zones`(deprecated config key) from `emqx_ctl conf show_keys` + +- [#11165](https://github.com/emqx/emqx/pull/11165) Remove `/configs/limiter` api from `swagger.json`, only the api documentation was removed, + and the `/configs/limiter` api functionalities have not been changed. + +- [#11166](https://github.com/emqx/emqx/pull/11166) Added 3 random SQL functions to the rule engine. + - random(): Generates a random number between 0 and 1 (0.0 =< X < 1.0). + - uuid_v4(): Generates a random UUID (version 4) string. + - uuid_v4_no_hyphen(): Generates a random UUID (version 4) string without hyphens. + +- [#11180](https://github.com/emqx/emqx/pull/11180) Adding a new configuration API `/configs`(GET/PUT) that supports to reload the hocon format configuration file. + +- [#11020](https://github.com/emqx/emqx/pull/11020) Upgraded emqtt dependency to avoid sensitive data leakage in the debug log. + +- [#11135](https://github.com/emqx/emqx/pull/11135) Improve time offset parser in rules engine and return uniform error codes. + +## Bug Fixes + +- [#11004](https://github.com/emqx/emqx/pull/11004) Do not allow wildcards for destination topic in rewrite rules. + +- [#11026](https://github.com/emqx/emqx/pull/11026) Addressed an inconsistency in the usage of 'div' and 'mod' operations within the rule engine. Previously, the 'div' operation was only usable as an infix operation and 'mod' could only be applied through a function call. With this change, both 'div' and 'mod' can be used via function call syntax and infix syntax. + +- [#11037](https://github.com/emqx/emqx/pull/11037) When starting an HTTP connector EMQX now returns a descriptive error in case the system is unable to connect to the remote target system. + +- [#11039](https://github.com/emqx/emqx/pull/11039) Fixed database number validation for Redis connector. Previously negative numbers were accepted as valid database numbers. + +- [#11074](https://github.com/emqx/emqx/pull/11074) Fix to adhere to Protocol spec MQTT-5.0 [MQTT-3.8.3-4]. + +- [#11094](https://github.com/emqx/emqx/pull/11094) Fixed an issue where connection errors in Kafka Producer would not be reported when reconnecting the bridge. + +- [#11103](https://github.com/emqx/emqx/pull/11103) Updated `erlcloud` dependency. + +- [#11106](https://github.com/emqx/emqx/pull/11106) Added a validation for the maximum number of pool workers of a bridge. + + Now the maximum amount is 1024 to avoid large memory consumption from an unreasonable number of workers. + +- [#11118](https://github.com/emqx/emqx/pull/11118) Ensure that validation errors in REST API responses are slightly less confusing. Now, if there are out-of-range errors, they will be presented as `{"value": 42, "reason": {"expected": "1..10"}, ...}`, replacing the previous usage of `expected_type` with `expected`. + +- [#11126](https://github.com/emqx/emqx/pull/11126) Rule metrics for async mode bridges will set failure counters correctly now. + +- [#11134](https://github.com/emqx/emqx/pull/11134) Fix the value of the uppercase `authorization` header is not obfuscated. + +- [#11139](https://github.com/emqx/emqx/pull/11139) The Redis connector has been refactored to its own Erlang application to improve the code structure. + +- [#11145](https://github.com/emqx/emqx/pull/11145) Add several fixes and improvements in Ekka and Mria. + + Ekka: + - improve cluster discovery log messages to consistently describe actual events + [Ekka PR](https://github.com/emqx/ekka/pull/204) + - remove deprecated cluster auto-clean configuration parameter (it has been moved to Mria) + [Ekka PR](https://github.com/emqx/ekka/pull/203) + + Mria: + - ping only running replicant nodes. Previously, `mria_lb` was trying to ping both stopped and running + replicant nodes, which might result in timeout errors. + [Mria PR](https://github.com/emqx/mria/pull/146) + - use `null_copies` storage when copying `$mria_rlog_sync` table. + This fix has no effect on EMQX for now, as `$mria_rlog_sync` is only used in `mria:sync_transaction/2,3,4`, + which is not utilized by EMQX. + [Mria PR](https://github.com/emqx/mria/pull/144) + +- [#11148](https://github.com/emqx/emqx/pull/11148) Fix when a node has left the cluster, other nodes still try to synchronize configuration update operations to it. + +- [#11150](https://github.com/emqx/emqx/pull/11150) Wait for Mria table when emqx_psk app is being started to ensure that + PSK data is synced to replicant nodes even if they don't have init PSK file. + +- [#11151](https://github.com/emqx/emqx/pull/11151) The MySQL connector has been refactored to its own Erlang application to improve the code structure. + +- [#11158](https://github.com/emqx/emqx/pull/11158) Wait for Mria table when the mnesia backend of retainer starts to avoid a possible error of the retainer when joining a cluster. + +- [#11162](https://github.com/emqx/emqx/pull/11162) Fixed an issue in webhook bridge where, in async query mode, HTTP status codes like 4XX and 5XX would be treated as successes in the bridge metrics. + +- [#11164](https://github.com/emqx/emqx/pull/11164) Reintroduced support for nested (i.e.: `${payload.a.b.c}`) placeholders for extracting data from rule action messages without the need for calling `json_decode(payload)` first. + +- [#11172](https://github.com/emqx/emqx/pull/11172) Fix the `payload` will be duplicated in the below situations: + - Use a `foreach` sentence without the `as` sub-expression and select all fields(use the `*` or omitted the `do` sub-expression) + + For example: + + `FOREACH payload.sensors FROM "t/#"` + - Select the `payload` field and all fields + + For example: + + `SELECT payload.sensors, * FROM "t/#"` + +- [#11174](https://github.com/emqx/emqx/pull/11174) Fixed the encoding of the `server` key coming from an ingress MQTT bridge. + + Before the fix, it was being encoded as a list of integers corresponding to the ASCII characters of the server string. From 4dfd3421e77bd4282435c4f5662bd02af720fafa Mon Sep 17 00:00:00 2001 From: Serge Tupchii Date: Wed, 5 Jul 2023 20:24:26 +0300 Subject: [PATCH 11/92] chore(emqx_authz): fix `rule()` type example Use `action()` type which is a correct and defined type. --- apps/emqx_authz/etc/acl.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_authz/etc/acl.conf b/apps/emqx_authz/etc/acl.conf index 9964dc7ba..dbeec6852 100644 --- a/apps/emqx_authz/etc/acl.conf +++ b/apps/emqx_authz/etc/acl.conf @@ -20,7 +20,7 @@ %% %% -type(permission() :: allow | deny). %% -%% -type(rule() :: {permission(), who(), access(), topics()} | {permission(), all}). +%% -type(rule() :: {permission(), who(), action(), topics()} | {permission(), all}). %%-------------------------------------------------------------------- {allow, {username, {re, "^dashboard$"}}, subscribe, ["$SYS/#"]}. From 966b2affc2dbdd83184174cbe6282b39b8f1d470 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 10:53:07 +0800 Subject: [PATCH 12/92] chore: allow handle */* or multiple values in Accept headers --- .../src/emqx_management.app.src | 2 +- .../src/emqx_mgmt_api_configs.erl | 29 +++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/src/emqx_management.app.src b/apps/emqx_management/src/emqx_management.app.src index 0e2c2646e..d6286f454 100644 --- a/apps/emqx_management/src/emqx_management.app.src +++ b/apps/emqx_management/src/emqx_management.app.src @@ -2,7 +2,7 @@ {application, emqx_management, [ {description, "EMQX Management API and CLI"}, % strict semver, bump manually! - {vsn, "5.0.25"}, + {vsn, "5.0.26"}, {modules, []}, {registered, [emqx_management_sup]}, {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 6991bb11c..80818b41a 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -337,9 +337,10 @@ config_reset(post, _Params, Req) -> configs(get, #{query_string := QueryStr, headers := Headers}, _Req) -> %% Should deprecated json v1 since 5.2.0 - case maps:get(<<"accept">>, Headers, <<"text/plain">>) of - <<"application/json">> -> get_configs_v1(QueryStr); - <<"text/plain">> -> get_configs_v2(QueryStr) + case find_suitable_accept(Headers, [<<"text/plain">>, <<"application/json">>]) of + {ok, <<"application/json">>} -> get_configs_v1(QueryStr); + {ok, <<"text/plain">>} -> get_configs_v2(QueryStr); + {error, _} = Error -> {400, #{code => 'INVALID_ACCEPT', message => ?ERR_MSG(Error)}} end; configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> case emqx_conf_cli:load_config(Conf, Mode) of @@ -348,6 +349,28 @@ configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> {error, Errors} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Errors)}} end. +find_suitable_accept(Headers, Perferences) -> + AcceptVal = maps:get(<<"accept">>, Headers, <<"*/*">>), + %% Multiple types, weighted with the quality value syntax: + %% Accept: text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8 + Accepts = lists:map( + fun(S) -> + [T | _] = binary:split(string:trim(S), <<";">>), + T + end, + re:split(AcceptVal, ",") + ), + case lists:member(<<"*/*">>, Accepts) of + true -> + {ok, lists:first(Perferences)}; + fales -> + Found = lists:filter(fun(Accept) -> lists:member(Accept, Accepts) end, Perferences), + case Found of + [] -> {error, no_suitalbe_accept}; + _ -> lists:first(Found) + end + end. + get_configs_v1(QueryStr) -> Node = maps:get(<<"node">>, QueryStr, node()), case From 71d1f805300c5200e38f36fb7697e32210c79148 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 11:30:38 +0800 Subject: [PATCH 13/92] test(api): cover the accept logic for `/configs` API --- .../src/emqx_mgmt_api_configs.erl | 9 +++--- .../test/emqx_mgmt_api_configs_SUITE.erl | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_configs.erl b/apps/emqx_management/src/emqx_mgmt_api_configs.erl index 80818b41a..f2e336d0f 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_configs.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_configs.erl @@ -122,6 +122,7 @@ schema("/configs") -> }} ] }, + 400 => emqx_dashboard_swagger:error_codes(['INVALID_ACCEPT']), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND']), 500 => emqx_dashboard_swagger:error_codes(['BAD_NODE']) } @@ -349,7 +350,7 @@ configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) -> {error, Errors} -> {400, #{code => 'UPDATE_FAILED', message => ?ERR_MSG(Errors)}} end. -find_suitable_accept(Headers, Perferences) -> +find_suitable_accept(Headers, Perferences) when is_list(Perferences), length(Perferences) > 0 -> AcceptVal = maps:get(<<"accept">>, Headers, <<"*/*">>), %% Multiple types, weighted with the quality value syntax: %% Accept: text/html, application/xhtml+xml, application/xml;q=0.9, image/webp, */*;q=0.8 @@ -362,12 +363,12 @@ find_suitable_accept(Headers, Perferences) -> ), case lists:member(<<"*/*">>, Accepts) of true -> - {ok, lists:first(Perferences)}; - fales -> + {ok, lists:nth(1, Perferences)}; + false -> Found = lists:filter(fun(Accept) -> lists:member(Accept, Accepts) end, Perferences), case Found of [] -> {error, no_suitalbe_accept}; - _ -> lists:first(Found) + _ -> {ok, lists:nth(1, Found)} end end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl index 0e54a3e22..43554c9ff 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl @@ -323,6 +323,36 @@ t_configs_key(_Config) -> ?assertEqual(<<"error">>, read_conf([<<"log">>, <<"console">>, <<"level">>])), ok. +t_get_configs_in_different_accept(_Config) -> + [Key | _] = lists:sort(emqx_conf_cli:keys()), + URI = emqx_mgmt_api_test_util:api_path(["configs?key=" ++ Key]), + Auth = emqx_mgmt_api_test_util:auth_header_(), + Request = fun(Accept) -> + Headers = [{"accept", Accept}, Auth], + case + emqx_mgmt_api_test_util:request_api(get, URI, [], Headers, [], #{return_all => true}) + of + {ok, {{_, Code, _}, RespHeaders, Body}} -> + Type = proplists:get_value("content-type", RespHeaders), + {Code, Type, Body}; + {error, {{_, Code, _}, RespHeaders, Body}} -> + Type = proplists:get_value("content-type", RespHeaders), + {Code, Type, Body} + end + end, + + %% returns text/palin if text/plain is acceptable + ?assertMatch({200, "text/plain", _}, Request(<<"text/plain">>)), + ?assertMatch({200, "text/plain", _}, Request(<<"*/*">>)), + ?assertMatch( + {200, "text/plain", _}, + Request(<<"application/json, application/xml;q=0.9, image/webp, */*;q=0.8">>) + ), + %% returns application/json if it only support it + ?assertMatch({200, "application/json", _}, Request(<<"application/json">>)), + %% returns error if it set to other type + ?assertMatch({400, "application/json", _}, Request(<<"application/xml">>)). + %% Helpers get_config(Name) -> From ccacb50393ae4f7dfb9e4f5e4462b04f51738c9a Mon Sep 17 00:00:00 2001 From: Rory Z <16801068+Rory-Z@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:08:10 +0800 Subject: [PATCH 14/92] ci: do not set latest for elixir image --- .github/workflows/build_and_push_docker_images.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 3b47150c2..3ee9b79c7 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -182,6 +182,7 @@ jobs: images: | ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }} flavor: | + latest=${{ matrix.elixir == 'no_elixir' }} suffix=${{ steps.pre-meta.outputs.img_suffix }} tags: | type=semver,pattern={{major}}.{{minor}},value=${{ needs.prepare.outputs.VERSION }} From 71afd1e3a41c70156b548b70533e452201a39f37 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 6 Jul 2023 10:03:07 +0200 Subject: [PATCH 15/92] fix(mix): use tag for emqx/ranch version instead of sha --- mix.exs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mix.exs b/mix.exs index 1e18ef8ef..1cdea809f 100644 --- a/mix.exs +++ b/mix.exs @@ -91,8 +91,7 @@ defmodule EMQXUmbrella.MixProject do {:cowlib, github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true}, # in conflict by cowboy_swagger and cowboy - {:ranch, - github: "emqx/ranch", ref: "de8ba2a00817c0a6eb1b8f20d6fb3e44e2c9a5aa", override: true}, + {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true}, # in conflict by grpc and eetcd {:gpb, "4.19.7", override: true, runtime: false}, {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}, From 4f80aa1bec78d8bf001ae2195dce1511cdf6d57b Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 6 Jul 2023 10:09:25 +0200 Subject: [PATCH 16/92] ci(macos): stop doing brew update --- .github/actions/package-macos/action.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 8615a433a..6b47ceafa 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -33,7 +33,6 @@ runs: HOMEBREW_NO_INSTALL_UPGRADE: 1 HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: 1 run: | - brew update brew install curl zip unzip coreutils openssl@1.1 echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH echo "/usr/local/bin" >> $GITHUB_PATH From 22b25fbbb35f73f55a5bdfb557cadc403461fb89 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 6 Jul 2023 12:14:45 +0200 Subject: [PATCH 17/92] feat(emqx_dashboard): include edition and version in swagger.json info --- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.erl | 8 +++++++- apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl | 9 +++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index e2909eca6..9cceacf3a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.24"}, + {vsn, "5.0.25"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_dashboard/src/emqx_dashboard.erl b/apps/emqx_dashboard/src/emqx_dashboard.erl index de35c43b3..8c56d8014 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard.erl @@ -49,7 +49,7 @@ start_listeners(Listeners) -> Authorization = {?MODULE, authorize}, GlobalSpec = #{ openapi => "3.0.0", - info => #{title => "EMQX API", version => ?EMQX_API_VERSION}, + info => #{title => emqx_api_name(), version => emqx_release_version()}, servers => [#{url => emqx_dashboard_swagger:base_path()}], components => #{ schemas => #{}, @@ -271,3 +271,9 @@ dynamic_dispatch() -> {emqx_mgmt_api_status:path(), emqx_mgmt_api_status, []}, {'_', emqx_dashboard_not_found, []} ]. + +emqx_api_name() -> + emqx_release:description() ++ " API". + +emqx_release_version() -> + emqx_release:version(). diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 1230316e0..f0b6db8ea 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -149,6 +149,15 @@ t_swagger_json(_Config) -> {ok, {{"HTTP/1.1", 200, "OK"}, _Headers, Body2}} = httpc:request(get, {Url, []}, [], [{body_format, binary}]), ?assertEqual(Body1, Body2), + ?assertMatch( + #{ + <<"info">> := #{ + <<"title">> := _, + <<"version">> := _ + } + }, + emqx_utils_json:decode(Body1) + ), ok. t_cli(_Config) -> From 8faaa86c0743933188db17892b3adf7de0048ad2 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 6 Jul 2023 14:25:36 +0200 Subject: [PATCH 18/92] chore: bump vsn for emqx and emqx_bridge --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 007c0e72a..928539f46 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.1"}, + {vsn, "5.1.2"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index ac1a3443f..07711da12 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.22"}, + {vsn, "0.1.23"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ From f2e4ad39b851b841f0e56797b51127b884a8adc3 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 6 Jul 2023 11:00:35 +0200 Subject: [PATCH 19/92] docs: update CONTRIBUTING.md to include information about change log --- CONTRIBUTING.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 272a602e9..259718c6f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,6 @@ You are welcome to submit any bugs, issues and feature requests on this repository. - ## Commit Message Guidelines We have very precise rules over how our git commit messages can be formatted. This leads to **more readable messages** that are easy to follow when looking through the **project history**. @@ -80,3 +79,13 @@ Just as in the **subject**, use the imperative, present tense: "change" not "cha The footer should contain any information about **Breaking Changes** and is also the place to reference GitHub issues that this commit **Closes**. **Breaking Changes** should start with the word `BREAKING CHANGE:` with a space or two newlines. The rest of the commit message is then used for this. + +## Changelog + +Changes affecting EMQX functionality shall be described in a separate markdown file under `changes` directory. + +File name pattern: `changes/{ce,ee}/(feat|perf|fix)-.en.md`, where: + +- `ce,ee`: Indicates whether given change affects community and enterprise edition (`ce`), or enterprise edition only (`ee`); for any change only one file is needed as enterprise edition absorbs all changes from the community edition automatically. When in doubts, one could consult [documentation](https://www.emqx.io/docs/en/latest/). Enterprise features have a corresponding "Tip" banner, see for example [here](https://www.emqx.io/docs/en/v5.1/data-integration/data-bridge-influxdb.html). +- `feat|perf|fix`: Whether the change is a new functionality (`feat`), performance improvement (`perf`), or a bug fix (`fix`). +- `PR-id`: Github pull request id. Since pull request id cannot be known before the PR is actually created, it's common to add change log entry in a separate commit. From e4504c89420c0b639b75ed2afc9e1ae566757463 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Thu, 6 Jul 2023 14:32:06 +0200 Subject: [PATCH 20/92] docs: align notation for change log file name pattern also add a note that we are using English only for change log --- .github/pull_request_template.md | 2 +- CONTRIBUTING.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 26a3cc5fc..a12aeb012 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,7 +10,7 @@ Please convert it to a draft if any of the following conditions are not met. Rev - [ ] Added tests for the changes - [ ] Changed lines covered in coverage report -- [ ] Change log has been added to `changes/{ce,ee}/(feat|perf|fix)-.en.md` files +- [ ] Change log has been added to `changes/(ce|ee)/(feat|perf|fix)-.en.md` files - [ ] For internal contributor: there is a jira ticket to track this change - [ ] If there should be document changes, a PR to emqx-docs.git is sent, or a jira ticket is created to follow up - [ ] Schema changes are backward compatible diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 259718c6f..6337034eb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -84,8 +84,9 @@ The footer should contain any information about **Breaking Changes** and is also Changes affecting EMQX functionality shall be described in a separate markdown file under `changes` directory. -File name pattern: `changes/{ce,ee}/(feat|perf|fix)-.en.md`, where: +File name pattern: `changes/(ce|ee)/(feat|perf|fix)-.en.md`, where: - `ce,ee`: Indicates whether given change affects community and enterprise edition (`ce`), or enterprise edition only (`ee`); for any change only one file is needed as enterprise edition absorbs all changes from the community edition automatically. When in doubts, one could consult [documentation](https://www.emqx.io/docs/en/latest/). Enterprise features have a corresponding "Tip" banner, see for example [here](https://www.emqx.io/docs/en/v5.1/data-integration/data-bridge-influxdb.html). - `feat|perf|fix`: Whether the change is a new functionality (`feat`), performance improvement (`perf`), or a bug fix (`fix`). - `PR-id`: Github pull request id. Since pull request id cannot be known before the PR is actually created, it's common to add change log entry in a separate commit. +- `en`: ISO 639-1 language code indicating the language the change log entry is written in. Right now we are only accepting entries in English. From f415af7225cbccb0736c9a54d59fe660628f7a9b Mon Sep 17 00:00:00 2001 From: dounix Date: Thu, 6 Jul 2023 10:59:13 -0400 Subject: [PATCH 21/92] feat: enable setting ssl common name template, documentation, default values.yaml for value ssl.Commonname to support vault-issuer vault-issuer (k8s clusterIssuer) requires CN to be set by default closes: emqx#11199 --- deploy/charts/emqx-enterprise/README.md | 1 + deploy/charts/emqx-enterprise/templates/certificate.yaml | 3 +++ deploy/charts/emqx-enterprise/values.yaml | 1 + deploy/charts/emqx/README.md | 1 + deploy/charts/emqx/templates/certificate.yaml | 3 +++ deploy/charts/emqx/values.yaml | 1 + 6 files changed, 10 insertions(+) diff --git a/deploy/charts/emqx-enterprise/README.md b/deploy/charts/emqx-enterprise/README.md index df3be6766..b11159c84 100644 --- a/deploy/charts/emqx-enterprise/README.md +++ b/deploy/charts/emqx-enterprise/README.md @@ -100,6 +100,7 @@ The following table lists the configurable parameters of the emqx chart and thei | `ssl.useExisting` | Use existing certificate or let cert-manager generate one | false | | `ssl.existingName` | Name of existing certificate | emqx-tls | | `ssl.dnsnames` | DNS name(s) for certificate to be generated | {} | +| `ssl.commonName` | Common name for or certificate to be generated | | | `ssl.issuer.name` | Issuer name for certificate generation | letsencrypt-dns | | `ssl.issuer.kind` | Issuer kind for certificate generation | ClusterIssuer | diff --git a/deploy/charts/emqx-enterprise/templates/certificate.yaml b/deploy/charts/emqx-enterprise/templates/certificate.yaml index 9a2ed969a..528f989a1 100644 --- a/deploy/charts/emqx-enterprise/templates/certificate.yaml +++ b/deploy/charts/emqx-enterprise/templates/certificate.yaml @@ -9,6 +9,9 @@ spec: issuerRef: name: {{ default "letsencrypt-staging" .Values.ssl.issuer.name }} kind: {{ default "ClusterIssuer" .Values.ssl.issuer.kind }} + {{- if .Values.ssl.commonName }} + commonName: {{ .Values.ssl.commonName }} + {{- end }} dnsNames: {{- range .Values.ssl.dnsnames }} - {{ . }} diff --git a/deploy/charts/emqx-enterprise/values.yaml b/deploy/charts/emqx-enterprise/values.yaml index 71569b9a3..412462854 100644 --- a/deploy/charts/emqx-enterprise/values.yaml +++ b/deploy/charts/emqx-enterprise/values.yaml @@ -237,6 +237,7 @@ ssl: useExisting: false existingName: emqx-tls dnsnames: [] + commonName: issuer: name: letsencrypt-dns kind: ClusterIssuer diff --git a/deploy/charts/emqx/README.md b/deploy/charts/emqx/README.md index 47ae89245..7c6ef122f 100644 --- a/deploy/charts/emqx/README.md +++ b/deploy/charts/emqx/README.md @@ -99,6 +99,7 @@ The following table lists the configurable parameters of the emqx chart and thei | `ssl.enabled` | Enable SSL support | false | | `ssl.useExisting` | Use existing certificate or let cert-manager generate one | false | | `ssl.existingName` | Name of existing certificate | emqx-tls | +| `ssl.commonName` | Common name for or certificate to be generated | | | `ssl.dnsnames` | DNS name(s) for certificate to be generated | {} | | `ssl.issuer.name` | Issuer name for certificate generation | letsencrypt-dns | | `ssl.issuer.kind` | Issuer kind for certificate generation | ClusterIssuer | diff --git a/deploy/charts/emqx/templates/certificate.yaml b/deploy/charts/emqx/templates/certificate.yaml index 9a2ed969a..528f989a1 100644 --- a/deploy/charts/emqx/templates/certificate.yaml +++ b/deploy/charts/emqx/templates/certificate.yaml @@ -9,6 +9,9 @@ spec: issuerRef: name: {{ default "letsencrypt-staging" .Values.ssl.issuer.name }} kind: {{ default "ClusterIssuer" .Values.ssl.issuer.kind }} + {{- if .Values.ssl.commonName }} + commonName: {{ .Values.ssl.commonName }} + {{- end }} dnsNames: {{- range .Values.ssl.dnsnames }} - {{ . }} diff --git a/deploy/charts/emqx/values.yaml b/deploy/charts/emqx/values.yaml index f7c6483fe..86fa44880 100644 --- a/deploy/charts/emqx/values.yaml +++ b/deploy/charts/emqx/values.yaml @@ -240,6 +240,7 @@ ssl: useExisting: false existingName: emqx-tls dnsnames: [] + commonName: issuer: name: letsencrypt-dns kind: ClusterIssuer From 18dec53d8b00060874eccae1061609f6f33cf55a Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 6 Jul 2023 15:08:00 +0800 Subject: [PATCH 22/92] feat: ensure data backends don't leak sensitive data --- apps/emqx_bridge_clickhouse/rebar.config | 2 +- apps/emqx_bridge_tdengine/rebar.config | 2 +- apps/emqx_mongodb/rebar.config | 2 +- apps/emqx_mongodb/src/emqx_mongodb.erl | 2 +- apps/emqx_mysql/rebar.config | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge_clickhouse/rebar.config b/apps/emqx_bridge_clickhouse/rebar.config index a8da74b43..98d889f41 100644 --- a/apps/emqx_bridge_clickhouse/rebar.config +++ b/apps/emqx_bridge_clickhouse/rebar.config @@ -1,6 +1,6 @@ %% -*- mode: erlang; -*- {erl_opts, [debug_info]}. -{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}} +{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3.1"}}} , {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} , {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/apps/emqx_bridge_tdengine/rebar.config b/apps/emqx_bridge_tdengine/rebar.config index 72ebca1db..97ccf918a 100644 --- a/apps/emqx_bridge_tdengine/rebar.config +++ b/apps/emqx_bridge_tdengine/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [debug_info]}. {deps, [ - {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}}, + {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.7"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}} diff --git a/apps/emqx_mongodb/rebar.config b/apps/emqx_mongodb/rebar.config index e8a7e281d..cfd7dc9be 100644 --- a/apps/emqx_mongodb/rebar.config +++ b/apps/emqx_mongodb/rebar.config @@ -3,5 +3,5 @@ {erl_opts, [debug_info]}. {deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} , {emqx_resource, {path, "../../apps/emqx_resource"}} - , {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.19"}}} + , {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.20"}}} ]}. diff --git a/apps/emqx_mongodb/src/emqx_mongodb.erl b/apps/emqx_mongodb/src/emqx_mongodb.erl index 4236517e2..dfa732a7b 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.erl +++ b/apps/emqx_mongodb/src/emqx_mongodb.erl @@ -424,7 +424,7 @@ init_worker_options([{auth_source, V} | R], Acc) -> init_worker_options([{username, V} | R], Acc) -> init_worker_options(R, [{login, V} | Acc]); init_worker_options([{password, V} | R], Acc) -> - init_worker_options(R, [{password, V} | Acc]); + init_worker_options(R, [{password, emqx_secret:wrap(V)} | Acc]); init_worker_options([{w_mode, V} | R], Acc) -> init_worker_options(R, [{w_mode, V} | Acc]); init_worker_options([{r_mode, V} | R], Acc) -> diff --git a/apps/emqx_mysql/rebar.config b/apps/emqx_mysql/rebar.config index 668e437f3..fc7f4df7a 100644 --- a/apps/emqx_mysql/rebar.config +++ b/apps/emqx_mysql/rebar.config @@ -3,7 +3,7 @@ {erl_opts, [debug_info]}. {deps, [ %% NOTE: mind ecpool version when updating eredis_cluster version - {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.3"}}}, + {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.4"}}}, {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}} ]}. From afe698962571e8f7e94156f10403e1f7e67ef35f Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 6 Jul 2023 15:19:34 +0800 Subject: [PATCH 23/92] chore: update changes --- changes/ee/feat-11207.en.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 changes/ee/feat-11207.en.md diff --git a/changes/ee/feat-11207.en.md b/changes/ee/feat-11207.en.md new file mode 100644 index 000000000..d1d4c1812 --- /dev/null +++ b/changes/ee/feat-11207.en.md @@ -0,0 +1,6 @@ +Update the dependent versions of multiple data bridges to enhance security and ensure that sensitive data will not be leaked. +Including: + - TDEngine + - MongoDB + - MySQL + - Clickhouse From a1d2b2ca5191016f2c745468f02c35b924ea05ae Mon Sep 17 00:00:00 2001 From: firest Date: Thu, 6 Jul 2023 16:08:24 +0800 Subject: [PATCH 24/92] chore: bump app versions --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_dashboard/src/emqx_dashboard.app.src | 2 +- apps/emqx_mongodb/src/emqx_mongodb.app.src | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 007c0e72a..928539f46 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.1"}, + {vsn, "5.1.2"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index ac1a3443f..07711da12 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.22"}, + {vsn, "0.1.23"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_dashboard/src/emqx_dashboard.app.src b/apps/emqx_dashboard/src/emqx_dashboard.app.src index e2909eca6..9cceacf3a 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard.app.src +++ b/apps/emqx_dashboard/src/emqx_dashboard.app.src @@ -2,7 +2,7 @@ {application, emqx_dashboard, [ {description, "EMQX Web Dashboard"}, % strict semver, bump manually! - {vsn, "5.0.24"}, + {vsn, "5.0.25"}, {modules, []}, {registered, [emqx_dashboard_sup]}, {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]}, diff --git a/apps/emqx_mongodb/src/emqx_mongodb.app.src b/apps/emqx_mongodb/src/emqx_mongodb.app.src index 56419e37b..00dcb0cfb 100644 --- a/apps/emqx_mongodb/src/emqx_mongodb.app.src +++ b/apps/emqx_mongodb/src/emqx_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_mongodb, [ {description, "EMQX MongoDB Connector"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, From 5c901a52bdb871115192c4df90bef76df51a9a2b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 5 Jul 2023 16:08:20 +0800 Subject: [PATCH 25/92] fix(coap): make username/password optinal in connection --- apps/emqx_gateway/src/emqx_gateway_utils.erl | 6 +- .../src/emqx_coap_channel.erl | 61 ++++++++----------- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_utils.erl b/apps/emqx_gateway/src/emqx_gateway_utils.erl index 634a02f03..eb4ce9fdf 100644 --- a/apps/emqx_gateway/src/emqx_gateway_utils.erl +++ b/apps/emqx_gateway/src/emqx_gateway_utils.erl @@ -46,7 +46,8 @@ global_chain/1, listener_chain/3, find_gateway_definitions/0, - plus_max_connections/2 + plus_max_connections/2, + random_clientid/1 ]). -export([stringfy/1]). @@ -631,3 +632,6 @@ ensure_gateway_loaded() -> emqx_gateway_mqttsn ] ). + +random_clientid(GwName) when is_atom(GwName) -> + iolist_to_binary([atom_to_list(GwName), "-", emqx_utils:gen_id()]). diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index 21066655e..a48589e36 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -486,46 +486,35 @@ enrich_conninfo( conninfo = ConnInfo } ) -> - %% FIXME: generate a random clientid if absent - case Queries of - #{<<"clientid">> := ClientId} -> - Interval = maps:get(interval, emqx_keepalive:info(KeepAlive)), - NConnInfo = ConnInfo#{ - clientid => ClientId, - proto_name => <<"CoAP">>, - proto_ver => <<"1">>, - clean_start => true, - keepalive => Interval, - expiry_interval => 0 - }, - {ok, Channel#channel{conninfo = NConnInfo}}; - _ -> - {error, "invalid queries", Channel} - end. + ClientId = + case maps:get(<<"clientid">>, Queries, undefined) of + undefined -> + emqx_gateway_utils:random_clientid(coap); + ClientId0 -> + ClientId0 + end, + Interval = maps:get(interval, emqx_keepalive:info(KeepAlive)), + NConnInfo = ConnInfo#{ + clientid => ClientId, + proto_name => <<"CoAP">>, + proto_ver => <<"1">>, + clean_start => true, + keepalive => Interval, + expiry_interval => 0 + }, + {ok, Channel#channel{conninfo = NConnInfo}}. enrich_clientinfo( {Queries, Msg}, - Channel = #channel{clientinfo = ClientInfo0} + Channel = #channel{conninfo = ConnInfo, clientinfo = ClientInfo0} ) -> - %% FIXME: - %% 1. generate a random clientid if absent; - %% 2. assgin username, password to `undefined` if absent - case Queries of - #{ - <<"username">> := UserName, - <<"password">> := Password, - <<"clientid">> := ClientId - } -> - ClientInfo = ClientInfo0#{ - username => UserName, - password => Password, - clientid => ClientId - }, - {ok, NClientInfo} = fix_mountpoint(Msg, ClientInfo), - {ok, Channel#channel{clientinfo = NClientInfo}}; - _ -> - {error, "invalid queries", Channel} - end. + ClientInfo = ClientInfo0#{ + clientid => maps:get(clientid, ConnInfo), + username => maps:get(<<"username">>, Queries, undefined), + password => maps:get(<<"password">>, Queries, undefined) + }, + {ok, NClientInfo} = fix_mountpoint(Msg, ClientInfo), + {ok, Channel#channel{clientinfo = NClientInfo}}. set_log_meta(_Input, #channel{clientinfo = #{clientid := ClientId}}) -> emqx_logger:set_metadata_clientid(ClientId), From 800b1545826e300d0078552afba832cb39c5c33e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 5 Jul 2023 16:24:15 +0800 Subject: [PATCH 26/92] test(coap): cover the params optional logic to create connection --- .../src/emqx_coap_channel.erl | 32 ++++++++--------- .../test/emqx_coap_SUITE.erl | 36 +++++++++++++++++++ .../src/emqx_exproto_channel.erl | 5 +-- 3 files changed, 52 insertions(+), 21 deletions(-) diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index a48589e36..bcafda41f 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -486,23 +486,21 @@ enrich_conninfo( conninfo = ConnInfo } ) -> - ClientId = - case maps:get(<<"clientid">>, Queries, undefined) of - undefined -> - emqx_gateway_utils:random_clientid(coap); - ClientId0 -> - ClientId0 - end, - Interval = maps:get(interval, emqx_keepalive:info(KeepAlive)), - NConnInfo = ConnInfo#{ - clientid => ClientId, - proto_name => <<"CoAP">>, - proto_ver => <<"1">>, - clean_start => true, - keepalive => Interval, - expiry_interval => 0 - }, - {ok, Channel#channel{conninfo = NConnInfo}}. + case Queries of + #{<<"clientid">> := ClientId} -> + Interval = maps:get(interval, emqx_keepalive:info(KeepAlive)), + NConnInfo = ConnInfo#{ + clientid => ClientId, + proto_name => <<"CoAP">>, + proto_ver => <<"1">>, + clean_start => true, + keepalive => Interval, + expiry_interval => 0 + }, + {ok, Channel#channel{conninfo = NConnInfo}}; + _ -> + {error, "clientid is required", Channel} + end. enrich_clientinfo( {Queries, Msg}, diff --git a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index 999493a79..95fdf8cca 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -133,6 +133,42 @@ t_connection(_) -> end, do(Action). +t_connection_optional_params(_) -> + UsernamePasswordAreOptional = + fun(Channel) -> + URI = + ?MQTT_PREFIX ++ + "/connection?clientid=client1", + Req = make_req(post), + {ok, created, Data} = do_request(Channel, URI, Req), + #coap_content{payload = Token0} = Data, + Token = binary_to_list(Token0), + + timer:sleep(100), + ?assertNotEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ), + + disconnection(Channel, Token), + + timer:sleep(100), + ?assertEqual( + [], + emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>) + ) + end, + ClientIdIsRequired = + fun(Channel) -> + URI = + ?MQTT_PREFIX ++ + "/connection", + Req = make_req(post), + {error, bad_request, _} = do_request(Channel, URI, Req) + end, + do(UsernamePasswordAreOptional), + do(ClientIdIsRequired). + t_connection_with_authn_failed(_) -> ChId = {{127, 0, 0, 1}, 5683}, {ok, Sock} = er_coap_udp_socket:start_link(), diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 2a144ffeb..5de597920 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -782,7 +782,7 @@ enrich_clientinfo(InClientInfo = #{proto_name := ProtoName}, ClientInfo) -> default_conninfo(ConnInfo) -> ConnInfo#{ clean_start => true, - clientid => anonymous_clientid(), + clientid => emqx_gateway_utils:random_clientid(exproto), username => undefined, conn_props => #{}, connected => true, @@ -822,6 +822,3 @@ proto_name_to_protocol(<<>>) -> exproto; proto_name_to_protocol(ProtoName) when is_binary(ProtoName) -> binary_to_atom(ProtoName). - -anonymous_clientid() -> - iolist_to_binary(["exproto-", emqx_utils:gen_id()]). From 6d2222318d2a61d7f5a46d5e0083bb339c54fd10 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 14:54:48 +0800 Subject: [PATCH 27/92] chore(coap): update the subscriptions_cnt stats in time --- apps/emqx_gateway_coap/src/emqx_coap_channel.erl | 8 ++++---- apps/emqx_gateway_coap/src/emqx_coap_session.erl | 8 ++++---- apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index bcafda41f..d95dc5bd2 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -118,8 +118,8 @@ info(ctx, #channel{ctx = Ctx}) -> Ctx. -spec stats(channel()) -> emqx_types:stats(). -stats(_) -> - []. +stats(#channel{session = Session}) -> + emqx_coap_session:stats(Session). -spec init(map(), map()) -> channel(). init( @@ -273,7 +273,7 @@ handle_call( SubReq, TempMsg, #{}, Session ), NSession = maps:get(session, Result), - {reply, {ok, {MountedTopic, NSubOpts}}, Channel#channel{session = NSession}}; + {reply, {ok, {MountedTopic, NSubOpts}}, [{event, updated}], Channel#channel{session = NSession}}; handle_call( {unsubscribe, Topic}, _From, @@ -300,7 +300,7 @@ handle_call( UnSubReq, TempMsg, #{}, Session ), NSession = maps:get(session, Result), - {reply, ok, Channel#channel{session = NSession}}; + {reply, ok, [{event, updated}], Channel#channel{session = NSession}}; handle_call(subscriptions, _From, Channel = #channel{session = Session}) -> Subs = emqx_coap_session:info(subscriptions, Session), {reply, {ok, maps:to_list(Subs)}, Channel}; diff --git a/apps/emqx_gateway_coap/src/emqx_coap_session.erl b/apps/emqx_gateway_coap/src/emqx_coap_session.erl index 5ae169675..562369e2f 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_session.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_session.erl @@ -117,15 +117,15 @@ info(inflight, _) -> info(inflight_cnt, _) -> 0; info(inflight_max, _) -> - 0; + infinity; info(retry_interval, _) -> infinity; info(mqueue, _) -> emqx_mqueue:init(#{max_len => 0, store_qos0 => false}); -info(mqueue_len, #session{transport_manager = TM}) -> - maps:size(TM); -info(mqueue_max, _) -> +info(mqueue_len, _) -> 0; +info(mqueue_max, _) -> + infinity; info(mqueue_dropped, _) -> 0; info(next_pkt_id, _) -> diff --git a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index 95fdf8cca..ce809184e 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -363,6 +363,9 @@ t_clients_subscription_api(_) -> maps:get(topic, SubsResp2) ), + %% check subscription_cnt + {200, #{subscriptions_cnt := 1}} = request(get, "/gateways/coap/clients/client1"), + {204, _} = request(delete, Path ++ "/tx"), {200, []} = request(get, Path) From bc1efdc4a7cfcaa720212b9cebbba6da25cde81b Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 14:59:01 +0800 Subject: [PATCH 28/92] chore: update changes --- changes/ce/fix-11206.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-11206.en.md diff --git a/changes/ce/fix-11206.en.md b/changes/ce/fix-11206.en.md new file mode 100644 index 000000000..e16b1e3f8 --- /dev/null +++ b/changes/ce/fix-11206.en.md @@ -0,0 +1 @@ +Make the username and password params of CoAP client to optional in connection mode. From 791b8ef671b2608c67cc8a230ccc532d9ded7447 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 16:31:47 +0800 Subject: [PATCH 29/92] chore: bump versions --- apps/emqx_gateway/src/emqx_gateway.app.src | 2 +- apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src | 2 +- apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway.app.src b/apps/emqx_gateway/src/emqx_gateway.app.src index bfcf4f2f2..6db4b0674 100644 --- a/apps/emqx_gateway/src/emqx_gateway.app.src +++ b/apps/emqx_gateway/src/emqx_gateway.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_gateway, [ {description, "The Gateway management application"}, - {vsn, "0.1.20"}, + {vsn, "0.1.21"}, {registered, []}, {mod, {emqx_gateway_app, []}}, {applications, [kernel, stdlib, emqx, emqx_authn, emqx_ctl]}, diff --git a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src index e03066695..a0cbc3e18 100644 --- a/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src +++ b/apps/emqx_gateway_coap/src/emqx_gateway_coap.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_coap, [ {description, "CoAP Gateway"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src index 66f9ddc89..5959eea3d 100644 --- a/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src +++ b/apps/emqx_gateway_exproto/src/emqx_gateway_exproto.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_exproto, [ {description, "ExProto Gateway"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [kernel, stdlib, grpc, emqx, emqx_gateway]}, {env, []}, From 31a240ba638de92051e2b893f1cbfc79a3c1e0c1 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 16:12:45 +0800 Subject: [PATCH 30/92] fix(lwm2m): update the stats correctly and timely --- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl | 8 ++++---- apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl | 4 ++-- apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl | 6 ++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index 77652744a..bbd2d4377 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -111,8 +111,8 @@ info(clientid, #channel{clientinfo = #{clientid := ClientId}}) -> info(ctx, #channel{ctx = Ctx}) -> Ctx. -stats(_) -> - []. +stats(#channel{session = Session}) -> + emqx_lwm2m_session:stats(Session). init( ConnInfo = #{ @@ -246,7 +246,7 @@ handle_call( Subs = emqx_lwm2m_session:info(subscriptions, Session), NSubs = maps:put(MountedTopic, NSubOpts, Subs), NSession = emqx_lwm2m_session:set_subscriptions(NSubs, Session), - {reply, {ok, {MountedTopic, NSubOpts}}, Channel#channel{session = NSession}}; + {reply, {ok, {MountedTopic, NSubOpts}}, [{event, updated}], Channel#channel{session = NSession}}; handle_call( {unsubscribe, Topic}, _From, @@ -269,7 +269,7 @@ handle_call( Subs = emqx_lwm2m_session:info(subscriptions, Session), NSubs = maps:remove(MountedTopic, Subs), NSession = emqx_lwm2m_session:set_subscriptions(NSubs, Session), - {reply, ok, Channel#channel{session = NSession}}; + {reply, ok, [{event, updated}], Channel#channel{session = NSession}}; handle_call(subscriptions, _From, Channel = #channel{session = Session}) -> Subs = maps:to_list(emqx_lwm2m_session:info(subscriptions, Session)), {reply, {ok, Subs}, Channel}; diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl index e267692a6..8c37d48e2 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_session.erl @@ -248,11 +248,11 @@ stats(subscriptions_max, _) -> stats(inflight_cnt, _) -> 0; stats(inflight_max, _) -> - 0; + infinity; stats(mqueue_len, _) -> 0; stats(mqueue_max, _) -> - 0; + infinity; stats(mqueue_dropped, _) -> 0; stats(next_pkt_id, _) -> diff --git a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl index 1779bf842..df1a5d2b3 100644 --- a/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl +++ b/apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl @@ -2486,6 +2486,12 @@ case100_subscription_api(Config) -> }, {201, _} = request(post, Path, SubReq), {200, _} = request(get, Path), + + %% check subscription_cnt + {200, #{subscriptions_cnt := 2}} = request( + get, "/gateways/lwm2m/clients/" ++ binary_to_list(ClientId) + ), + {204, _} = request(delete, Path ++ "/tx"), {200, [InitSub]} = request(get, Path). From b28909deb64e2561bf3dd92ecd91623b169bd503 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 16:17:11 +0800 Subject: [PATCH 31/92] chore: update changes --- changes/ce/fix-11208.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-11208.en.md diff --git a/changes/ce/fix-11208.en.md b/changes/ce/fix-11208.en.md new file mode 100644 index 000000000..56d5a398a --- /dev/null +++ b/changes/ce/fix-11208.en.md @@ -0,0 +1 @@ +Fix the issue of abnormal data statistics for LwM2M client. From d3d2c8a5329fdcb7e4c75c3d6ebf2bd57bd522c3 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 16:38:16 +0800 Subject: [PATCH 32/92] chore(lwm2m): bump version --- apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src index 3a1e3fc62..db7cd665f 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src +++ b/apps/emqx_gateway_lwm2m/src/emqx_gateway_lwm2m.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_lwm2m, [ {description, "LwM2M Gateway"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway, emqx_gateway_coap]}, {env, []}, From 221f6eba0611c7424bc80e9f6207ece7a2fdb7da Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Thu, 6 Jul 2023 23:56:15 +0800 Subject: [PATCH 33/92] fix: bad tnx-id when rejoin cluster --- apps/emqx/src/emqx_app.erl | 14 +----- apps/emqx_conf/src/emqx_cluster_rpc.erl | 60 ++++++++++++++++++------- apps/emqx_conf/src/emqx_conf.app.src | 2 +- apps/emqx_conf/src/emqx_conf_app.erl | 27 ++++++----- apps/emqx_conf/src/emqx_conf_schema.erl | 6 +-- apps/emqx_conf/src/emqx_conf_sup.erl | 10 ++--- 6 files changed, 67 insertions(+), 52 deletions(-) diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index ffb4e3d1e..038c93283 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -25,9 +25,7 @@ get_description/0, get_release/0, set_config_loader/1, - get_config_loader/0, - set_init_tnx_id/1, - get_init_tnx_id/0 + get_config_loader/0 ]). -include("logger.hrl"). @@ -65,16 +63,6 @@ set_config_loader(Module) when is_atom(Module) -> get_config_loader() -> application:get_env(emqx, config_loader, emqx). -%% @doc Set the transaction id from which this node should start applying after boot. -%% The transaction ID is received from the core node which we just copied the latest -%% config from. -set_init_tnx_id(TnxId) -> - application:set_env(emqx, cluster_rpc_init_tnx_id, TnxId). - -%% @doc Get the transaction id from which this node should start applying after boot. -get_init_tnx_id() -> - application:get_env(emqx, cluster_rpc_init_tnx_id, -1). - maybe_load_config() -> case get_config_loader() of emqx -> diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 599b8474b..003851420 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -17,7 +17,7 @@ -behaviour(gen_server). %% API --export([start_link/0, mnesia/1]). +-export([start_link/1, mnesia/1]). %% Note: multicall functions are statically checked by %% `emqx_bapi_trans' and `emqx_bpapi_static_checks' modules. Don't @@ -29,7 +29,8 @@ status/0, skip_failed_commit/1, fast_forward_to_commit/2, - on_mria_stop/1 + on_mria_stop/1, + wait_for_cluster_rpc/0 ]). -export([ commit/2, @@ -62,6 +63,10 @@ -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). + +start_link() -> + start_link(-1). + -endif. -boot_mnesia({mnesia, [boot]}). @@ -107,11 +112,11 @@ mnesia(boot) -> {attributes, record_info(fields, cluster_rpc_commit)} ]). -start_link() -> - start_link(node(), ?MODULE, get_retry_ms()). +start_link(TnxId) -> + start_link(TnxId, node(), ?MODULE, get_retry_ms()). -start_link(Node, Name, RetryMs) -> - case gen_server:start_link({local, Name}, ?MODULE, [Node, RetryMs], []) of +start_link(TnxId, Node, Name, RetryMs) -> + case gen_server:start_link({local, Name}, ?MODULE, [TnxId, Node, RetryMs], []) of {ok, Pid} -> {ok, Pid}; {error, {already_started, Pid}} -> @@ -276,29 +281,46 @@ on_mria_stop(leave) -> on_mria_stop(_) -> ok. +wait_for_cluster_rpc() -> + %% Workaround for https://github.com/emqx/mria/issues/94: + Msg1 = #{msg => "wait_for_cluster_rpc_shard"}, + case mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1500) of + ok -> ?SLOG(info, Msg1#{result => ok}); + Error0 -> ?SLOG(error, Msg1#{result => Error0}) + end, + Msg2 = #{msg => "wait_for_cluster_rpc_tables"}, + case mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]) of + ok -> ?SLOG(info, Msg2#{result => ok}); + Error1 -> ?SLOG(error, Msg2#{result => Error1}) + end, + ok. + %%%=================================================================== %%% gen_server callbacks %%%=================================================================== %% @private -init([Node, RetryMs]) -> +init([TnxId, Node, RetryMs]) -> register_mria_stop_cb(fun ?MODULE:on_mria_stop/1), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs, is_leaving => false}, - %% The init transaction ID is set in emqx_conf_app after - %% it has fetched the latest config from one of the core nodes - TnxId = emqx_app:get_init_tnx_id(), - ok = maybe_init_tnx_id(Node, TnxId), %% Now continue with the normal catch-up process %% That is: apply the missing transactions after the config %% was copied until now. - {ok, State, {continue, ?CATCH_UP}}. + {ok, State, {continue, {?CATCH_UP, TnxId}}}. %% @private -handle_continue(?CATCH_UP, State) -> +handle_continue({?CATCH_UP, TnxId}, State = #{node := Node}) -> %% emqx app must be started before %% trying to catch up the rpc commit logs ok = wait_for_emqx_ready(), + ok = wait_for_cluster_rpc(), + %% The init transaction ID is set in emqx_conf_app after + %% it has fetched the latest config from one of the core nodes + ok = maybe_init_tnx_id(Node, TnxId), + {noreply, State, catch_up(State)}; +%% @private +handle_continue(?CATCH_UP, State) -> {noreply, State, catch_up(State)}. handle_call(reset, _From, State) -> @@ -388,7 +410,8 @@ read_next_mfa(Node) -> }), TnxId; [#cluster_rpc_commit{tnx_id = LastAppliedID}] -> - LastAppliedID + 1 + OldestId = get_oldest_mfa_id(), + max(LastAppliedID + 1, OldestId) end, case mnesia:read(?CLUSTER_MFA, NextId) of [] -> caught_up; @@ -404,8 +427,7 @@ do_fast_forward_to_commit(ToTnxId, State = #{node := Node}) -> true -> NodeId; false -> - {atomic, LatestId} = transaction(fun ?MODULE:get_cluster_tnx_id/0, []), - case LatestId =< NodeId of + case latest_tnx_id() =< NodeId of true -> NodeId; false -> @@ -420,6 +442,12 @@ get_cluster_tnx_id() -> Id -> Id end. +get_oldest_mfa_id() -> + case mnesia:first(?CLUSTER_MFA) of + '$end_of_table' -> 0; + Id -> Id + end. + %% The entry point of a config change transaction. init_mfa(Node, MFA) -> mnesia:write_lock_table(?CLUSTER_MFA), diff --git a/apps/emqx_conf/src/emqx_conf.app.src b/apps/emqx_conf/src/emqx_conf.app.src index d8ee672f3..3c1e5592f 100644 --- a/apps/emqx_conf/src/emqx_conf.app.src +++ b/apps/emqx_conf/src/emqx_conf.app.src @@ -1,6 +1,6 @@ {application, emqx_conf, [ {description, "EMQX configuration management"}, - {vsn, "0.1.23"}, + {vsn, "0.1.24"}, {registered, []}, {mod, {emqx_conf_app, []}}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index c92c28971..08fe73e69 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -31,16 +31,17 @@ -define(DEFAULT_INIT_TXN_ID, -1). start(_StartType, _StartArgs) -> - try - ok = init_conf() - catch - C:E:St -> - %% logger is not quite ready. - io:format(standard_error, "Failed to load config~n~p~n~p~n~p~n", [C, E, St]), - init:stop(1) - end, + {ok, TnxId} = + try + {ok, _} = init_conf() + catch + C:E:St -> + %% logger is not quite ready. + io:format(standard_error, "Failed to load config~n~p~n~p~n~p~n", [C, E, St]), + init:stop(1) + end, ok = emqx_config_logger:refresh_config(), - emqx_conf_sup:start_link(). + emqx_conf_sup:start_link(TnxId). stop(_State) -> ok. @@ -112,12 +113,10 @@ init_load_done() -> emqx_app:get_config_loader() =/= emqx. init_conf() -> - %% Workaround for https://github.com/emqx/mria/issues/94: - _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000), - _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]), + emqx_cluster_rpc:wait_for_cluster_rpc(), {ok, TnxId} = sync_cluster_conf(), - _ = emqx_app:set_init_tnx_id(TnxId), - ok = init_load(). + ok = init_load(), + {ok, TnxId}. cluster_nodes() -> mria:cluster_nodes(cores) -- [node()]. diff --git a/apps/emqx_conf/src/emqx_conf_schema.erl b/apps/emqx_conf/src/emqx_conf_schema.erl index d5d92920d..816e2f454 100644 --- a/apps/emqx_conf/src/emqx_conf_schema.erl +++ b/apps/emqx_conf/src/emqx_conf_schema.erl @@ -684,10 +684,10 @@ fields("cluster_call") -> )}, {"max_history", sc( - range(1, 500), + range(100, 10240), #{ desc => ?DESC(cluster_call_max_history), - default => 100 + default => 1024 } )}, {"cleanup_interval", @@ -695,7 +695,7 @@ fields("cluster_call") -> emqx_schema:duration(), #{ desc => ?DESC(cluster_call_cleanup_interval), - default => <<"5m">> + default => <<"24h">> } )} ]; diff --git a/apps/emqx_conf/src/emqx_conf_sup.erl b/apps/emqx_conf/src/emqx_conf_sup.erl index 6a3d795ae..d224db28e 100644 --- a/apps/emqx_conf/src/emqx_conf_sup.erl +++ b/apps/emqx_conf/src/emqx_conf_sup.erl @@ -18,16 +18,16 @@ -behaviour(supervisor). --export([start_link/0]). +-export([start_link/1]). -export([init/1]). -define(SERVER, ?MODULE). -start_link() -> - supervisor:start_link({local, ?SERVER}, ?MODULE, []). +start_link(TnxId) -> + supervisor:start_link({local, ?SERVER}, ?MODULE, [TnxId]). -init([]) -> +init([TnxId]) -> SupFlags = #{ strategy => one_for_all, intensity => 10, @@ -35,7 +35,7 @@ init([]) -> }, ChildSpecs = [ - child_spec(emqx_cluster_rpc, []), + child_spec(emqx_cluster_rpc, [TnxId]), child_spec(emqx_cluster_rpc_cleaner, []) ], {ok, {SupFlags, ChildSpecs}}. From 9f57ba510ebe6275a82457a74d97f04829715a8f Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 7 Jul 2023 08:10:33 +0800 Subject: [PATCH 34/92] chore: add 11214 changelog --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 13 ++++++++----- changes/ce/fix-11214.en.md | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 changes/ce/fix-11214.en.md diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 003851420..072b92347 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -60,6 +60,11 @@ -export_type([tnx_id/0, succeed_num/0]). +-boot_mnesia({mnesia, [boot]}). + +-include_lib("emqx/include/logger.hrl"). +-include("emqx_conf.hrl"). + -ifdef(TEST). -compile(export_all). -compile(nowarn_export_all). @@ -67,13 +72,11 @@ start_link() -> start_link(-1). +start_link(Node, Name, RetryMs) -> + start_link(-1, Node, Name, RetryMs). + -endif. --boot_mnesia({mnesia, [boot]}). - --include_lib("emqx/include/logger.hrl"). --include("emqx_conf.hrl"). - -define(INITIATE(MFA), {initiate, MFA}). -define(CATCH_UP, catch_up). -define(TIMEOUT, timer:minutes(1)). diff --git a/changes/ce/fix-11214.en.md b/changes/ce/fix-11214.en.md new file mode 100644 index 000000000..35a33970a --- /dev/null +++ b/changes/ce/fix-11214.en.md @@ -0,0 +1 @@ +Fix a bug where node configuration may fail to synchronize correctly when joining the cluster. From f7513b900ae75b4e1f6c3b72ed8da268ed0f7bc2 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 7 Jul 2023 11:05:26 +0800 Subject: [PATCH 35/92] fix: set load config done after update tnx_id --- apps/emqx_conf/src/emqx_cluster_rpc.erl | 29 +++++++++---------------- apps/emqx_conf/src/emqx_conf_app.erl | 27 ++++++++++++----------- apps/emqx_conf/src/emqx_conf_sup.erl | 10 ++++----- 3 files changed, 29 insertions(+), 37 deletions(-) diff --git a/apps/emqx_conf/src/emqx_cluster_rpc.erl b/apps/emqx_conf/src/emqx_cluster_rpc.erl index 072b92347..bb154f8b5 100644 --- a/apps/emqx_conf/src/emqx_cluster_rpc.erl +++ b/apps/emqx_conf/src/emqx_cluster_rpc.erl @@ -17,7 +17,7 @@ -behaviour(gen_server). %% API --export([start_link/1, mnesia/1]). +-export([start_link/0, mnesia/1]). %% Note: multicall functions are statically checked by %% `emqx_bapi_trans' and `emqx_bpapi_static_checks' modules. Don't @@ -30,7 +30,8 @@ skip_failed_commit/1, fast_forward_to_commit/2, on_mria_stop/1, - wait_for_cluster_rpc/0 + wait_for_cluster_rpc/0, + maybe_init_tnx_id/2 ]). -export([ commit/2, @@ -69,12 +70,6 @@ -compile(export_all). -compile(nowarn_export_all). -start_link() -> - start_link(-1). - -start_link(Node, Name, RetryMs) -> - start_link(-1, Node, Name, RetryMs). - -endif. -define(INITIATE(MFA), {initiate, MFA}). @@ -115,11 +110,11 @@ mnesia(boot) -> {attributes, record_info(fields, cluster_rpc_commit)} ]). -start_link(TnxId) -> - start_link(TnxId, node(), ?MODULE, get_retry_ms()). +start_link() -> + start_link(node(), ?MODULE, get_retry_ms()). -start_link(TnxId, Node, Name, RetryMs) -> - case gen_server:start_link({local, Name}, ?MODULE, [TnxId, Node, RetryMs], []) of +start_link(Node, Name, RetryMs) -> + case gen_server:start_link({local, Name}, ?MODULE, [Node, RetryMs], []) of {ok, Pid} -> {ok, Pid}; {error, {already_started, Pid}} -> @@ -303,26 +298,22 @@ wait_for_cluster_rpc() -> %%%=================================================================== %% @private -init([TnxId, Node, RetryMs]) -> +init([Node, RetryMs]) -> register_mria_stop_cb(fun ?MODULE:on_mria_stop/1), {ok, _} = mnesia:subscribe({table, ?CLUSTER_MFA, simple}), State = #{node => Node, retry_interval => RetryMs, is_leaving => false}, %% Now continue with the normal catch-up process %% That is: apply the missing transactions after the config %% was copied until now. - {ok, State, {continue, {?CATCH_UP, TnxId}}}. + {ok, State, {continue, {?CATCH_UP, init}}}. %% @private -handle_continue({?CATCH_UP, TnxId}, State = #{node := Node}) -> +handle_continue({?CATCH_UP, init}, State) -> %% emqx app must be started before %% trying to catch up the rpc commit logs ok = wait_for_emqx_ready(), ok = wait_for_cluster_rpc(), - %% The init transaction ID is set in emqx_conf_app after - %% it has fetched the latest config from one of the core nodes - ok = maybe_init_tnx_id(Node, TnxId), {noreply, State, catch_up(State)}; -%% @private handle_continue(?CATCH_UP, State) -> {noreply, State, catch_up(State)}. diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 08fe73e69..3c9af9393 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -31,17 +31,16 @@ -define(DEFAULT_INIT_TXN_ID, -1). start(_StartType, _StartArgs) -> - {ok, TnxId} = - try - {ok, _} = init_conf() - catch - C:E:St -> - %% logger is not quite ready. - io:format(standard_error, "Failed to load config~n~p~n~p~n~p~n", [C, E, St]), - init:stop(1) - end, + try + ok = init_conf() + catch + C:E:St -> + %% logger is not quite ready. + io:format(standard_error, "Failed to load config~n~p~n~p~n~p~n", [C, E, St]), + init:stop(1) + end, ok = emqx_config_logger:refresh_config(), - emqx_conf_sup:start_link(TnxId). + emqx_conf_sup:start_link(). stop(_State) -> ok. @@ -94,10 +93,12 @@ sync_data_from_node() -> %% Internal functions %% ------------------------------------------------------------------------------ -init_load() -> +init_load(TnxId) -> case emqx_app:get_config_loader() of Module when Module == emqx; Module == emqx_conf -> ok = emqx_config:init_load(emqx_conf:schema_module()), + %% Set load config done after update(init) tnx_id. + ok = emqx_cluster_rpc:maybe_init_tnx_id(node(), TnxId), ok = emqx_app:set_config_loader(emqx_conf), ok; Module -> @@ -115,8 +116,8 @@ init_load_done() -> init_conf() -> emqx_cluster_rpc:wait_for_cluster_rpc(), {ok, TnxId} = sync_cluster_conf(), - ok = init_load(), - {ok, TnxId}. + ok = init_load(TnxId), + ok. cluster_nodes() -> mria:cluster_nodes(cores) -- [node()]. diff --git a/apps/emqx_conf/src/emqx_conf_sup.erl b/apps/emqx_conf/src/emqx_conf_sup.erl index d224db28e..6a3d795ae 100644 --- a/apps/emqx_conf/src/emqx_conf_sup.erl +++ b/apps/emqx_conf/src/emqx_conf_sup.erl @@ -18,16 +18,16 @@ -behaviour(supervisor). --export([start_link/1]). +-export([start_link/0]). -export([init/1]). -define(SERVER, ?MODULE). -start_link(TnxId) -> - supervisor:start_link({local, ?SERVER}, ?MODULE, [TnxId]). +start_link() -> + supervisor:start_link({local, ?SERVER}, ?MODULE, []). -init([TnxId]) -> +init([]) -> SupFlags = #{ strategy => one_for_all, intensity => 10, @@ -35,7 +35,7 @@ init([TnxId]) -> }, ChildSpecs = [ - child_spec(emqx_cluster_rpc, [TnxId]), + child_spec(emqx_cluster_rpc, []), child_spec(emqx_cluster_rpc_cleaner, []) ], {ok, {SupFlags, ChildSpecs}}. From d31a305bcb7bbd892b4fbbb6bfe6c85fe3fecfd8 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 29 Jun 2023 15:49:16 +0800 Subject: [PATCH 36/92] refactor(hstreamdb): HStreamDB bridge to its own application --- apps/emqx_bridge_hstreamdb/rebar.config | 11 +++++++++++ .../src/emqx_bridge_hstreamdb.app.src | 2 +- .../src/emqx_bridge_hstreamdb.erl | 4 ++-- .../src/emqx_bridge_hstreamdb_connector.erl | 2 +- .../test/emqx_bridge_hstreamdb_SUITE.erl | 2 +- .../test/emqx_connector_hstreamdb_SUITE.erl | 2 +- lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl | 8 ++++---- lib-ee/emqx_ee_connector/rebar.config | 1 - ...ge_hstreamdb.hocon => emqx_bridge_hstreamdb.hocon} | 2 +- ...db.hocon => emqx_bridge_hstreamdb_connector.hocon} | 2 +- 10 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 apps/emqx_bridge_hstreamdb/rebar.config rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl => apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl (95%) rename lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl => apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl (99%) rename lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl => apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl (91%) rename lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl => apps/emqx_bridge_hstreamdb/test/emqx_connector_hstreamdb_SUITE.erl (90%) rename rel/i18n/{emqx_ee_bridge_hstreamdb.hocon => emqx_bridge_hstreamdb.hocon} (97%) rename rel/i18n/{emqx_ee_connector_hstreamdb.hocon => emqx_bridge_hstreamdb_connector.hocon} (94%) diff --git a/apps/emqx_bridge_hstreamdb/rebar.config b/apps/emqx_bridge_hstreamdb/rebar.config new file mode 100644 index 000000000..d9fafe205 --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/rebar.config @@ -0,0 +1,11 @@ +%% -*- mode: erlang -*- +{erl_opts, [debug_info]}. +{deps, [ + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, + {emqx, {path, "../../apps/emqx"}}, + {emqx_utils, {path, "../../apps/emqx_utils"}} +]}. + +{shell, [ + {apps, [emqx_bridge_hstreamdb]} +]}. diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index 1cb3742b3..32fbc29ac 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_hstreamdb, [ {description, "EMQX Enterprise HStreamDB Bridge"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [kernel, stdlib]}, {env, []}, diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl similarity index 95% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl rename to apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl index 13a70e7c7..dacfc3633 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge_hstreamdb). +-module(emqx_bridge_hstreamdb). -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). @@ -66,7 +66,7 @@ fields("get") -> field(connector) -> mk( - hoconsc:union([binary(), ref(emqx_ee_connector_hstreamdb, config)]), + hoconsc:union([binary(), ref(emqx_bridge_hstreamdb_connector, config)]), #{ required => true, example => <<"hstreamdb:demo">>, diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl similarity index 99% rename from lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl rename to apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index 70eca83d7..c2a210271 100644 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector_hstreamdb.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -1,7 +1,7 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_hstreamdb). +-module(emqx_bridge_hstreamdb_connector). -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). diff --git a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl similarity index 91% rename from lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl rename to apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl index 867b09f32..4b12beed7 100644 --- a/lib-ee/emqx_ee_bridge/test/ee_bridge_hstreamdb_SUITE.erl +++ b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(ee_bridge_hstreamdb_SUITE). +-module(emqx_bridge_hstreamdb_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_connector_hstreamdb_SUITE.erl similarity index 90% rename from lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl rename to apps/emqx_bridge_hstreamdb/test/emqx_connector_hstreamdb_SUITE.erl index ad49d9f62..09ba487f7 100644 --- a/lib-ee/emqx_ee_connector/test/emqx_ee_connector_hstreamdb_SUITE.erl +++ b/apps/emqx_bridge_hstreamdb/test/emqx_connector_hstreamdb_SUITE.erl @@ -2,7 +2,7 @@ %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_connector_hstreamdb_SUITE). +-module(emqx_connector_hstreamdb_SUITE). -compile(nowarn_export_all). -compile(export_all). diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl index 66f0dc3b4..17da77680 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl @@ -30,7 +30,7 @@ api_schemas(Method) -> api_ref(emqx_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"), api_ref(emqx_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"), api_ref(emqx_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"), - api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method), + api_ref(emqx_bridge_hstreamdb, <<"hstreamdb">>, Method), api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"), api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"), api_ref(emqx_bridge_redis, <<"redis_single">>, Method ++ "_single"), @@ -54,7 +54,7 @@ schema_modules() -> [ emqx_bridge_kafka, emqx_bridge_cassandra, - emqx_ee_bridge_hstreamdb, + emqx_bridge_hstreamdb, emqx_bridge_gcp_pubsub, emqx_bridge_influxdb, emqx_bridge_mongodb, @@ -93,7 +93,7 @@ resource_type(kafka_consumer) -> emqx_bridge_kafka_impl_consumer; %% to hocon; keeping this as just `kafka' for backwards compatibility. resource_type(kafka) -> emqx_bridge_kafka_impl_producer; resource_type(cassandra) -> emqx_bridge_cassandra_connector; -resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb; +resource_type(hstreamdb) -> emqx_bridge_hstreamdb_connector; resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_impl_producer; resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer; resource_type(mongodb_rs) -> emqx_bridge_mongodb_connector; @@ -123,7 +123,7 @@ fields(bridges) -> [ {hstreamdb, mk( - hoconsc:map(name, ref(emqx_ee_bridge_hstreamdb, "config")), + hoconsc:map(name, ref(emqx_bridge_hstreamdb, "config")), #{ desc => <<"HStreamDB Bridge Config">>, required => false diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config index ee1d4e500..1f52a4f03 100644 --- a/lib-ee/emqx_ee_connector/rebar.config +++ b/lib-ee/emqx_ee_connector/rebar.config @@ -1,7 +1,6 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/rel/i18n/emqx_ee_bridge_hstreamdb.hocon b/rel/i18n/emqx_bridge_hstreamdb.hocon similarity index 97% rename from rel/i18n/emqx_ee_bridge_hstreamdb.hocon rename to rel/i18n/emqx_bridge_hstreamdb.hocon index cb43d483a..d9e7f1561 100644 --- a/rel/i18n/emqx_ee_bridge_hstreamdb.hocon +++ b/rel/i18n/emqx_bridge_hstreamdb.hocon @@ -1,4 +1,4 @@ -emqx_ee_bridge_hstreamdb { +emqx_bridge_hstreamdb { config_direction.desc: """The direction of this bridge, MUST be 'egress'""" diff --git a/rel/i18n/emqx_ee_connector_hstreamdb.hocon b/rel/i18n/emqx_bridge_hstreamdb_connector.hocon similarity index 94% rename from rel/i18n/emqx_ee_connector_hstreamdb.hocon rename to rel/i18n/emqx_bridge_hstreamdb_connector.hocon index f6838297f..001340e9c 100644 --- a/rel/i18n/emqx_ee_connector_hstreamdb.hocon +++ b/rel/i18n/emqx_bridge_hstreamdb_connector.hocon @@ -1,4 +1,4 @@ -emqx_ee_connector_hstreamdb { +emqx_bridge_hstreamdb_connector { config.desc: """HStreamDB connection config""" From b9bfdfd5837445ec30f78858293cec16f268afdb Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 3 Jul 2023 11:13:25 +0800 Subject: [PATCH 37/92] chore: compatibility with hstreamdb v0.15.0 --- apps/emqx_bridge_hstreamdb/rebar.config | 2 +- mix.exs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/rebar.config b/apps/emqx_bridge_hstreamdb/rebar.config index d9fafe205..9a70b55f9 100644 --- a/apps/emqx_bridge_hstreamdb/rebar.config +++ b/apps/emqx_bridge_hstreamdb/rebar.config @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {erl_opts, [debug_info]}. {deps, [ - {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}}, + {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.3.1+v0.12.0"}}}, {emqx, {path, "../../apps/emqx"}}, {emqx_utils, {path, "../../apps/emqx_utils"}} ]}. diff --git a/mix.exs b/mix.exs index 1cdea809f..99aacdf1d 100644 --- a/mix.exs +++ b/mix.exs @@ -195,7 +195,7 @@ defmodule EMQXUmbrella.MixProject do defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do [ - {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"}, + {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.3.1+v0.12.0"}, {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.11", override: true}, {:wolff, github: "kafka4beam/wolff", tag: "1.7.6"}, {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true}, From 1587f038a5c0464f1772b8119b3e897deae226c9 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 3 Jul 2023 18:45:22 +0800 Subject: [PATCH 38/92] feat: hstreamdb bridge with batch query --- .../src/emqx_bridge_hstreamdb.erl | 87 +++++---- .../src/emqx_bridge_hstreamdb_connector.erl | 177 ++++++++++-------- changes/ee/feat-10203.en.md | 1 + rel/i18n/emqx_bridge_hstreamdb.hocon | 14 +- .../emqx_bridge_hstreamdb_connector.hocon | 42 +++-- scripts/spellcheck/dicts/emqx.txt | 4 + 6 files changed, 181 insertions(+), 144 deletions(-) create mode 100644 changes/ee/feat-10203.en.md diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl index dacfc3633..7052e0120 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.erl @@ -32,16 +32,31 @@ conn_bridge_examples(Method) -> } ]. -values(_Method) -> +values(get) -> + values(post); +values(put) -> + values(post); +values(post) -> #{ - type => hstreamdb, + type => <<"hstreamdb">>, name => <<"demo">>, - connector => <<"hstreamdb:connector">>, - enable => true, - direction => egress, - local_topic => <<"local/topic/#">>, - payload => <<"${payload}">> - }. + direction => <<"egress">>, + url => <<"http://127.0.0.1:6570">>, + stream => <<"stream">>, + %% raw HRecord + record_template => + <<"{ \"temperature\": ${payload.temperature}, \"humidity\": ${payload.humidity} }">>, + pool_size => 8, + %% grpc_timeout => <<"1m">> + resource_opts => #{ + query_mode => sync, + batch_size => 100, + batch_time => <<"20ms">> + }, + ssl => #{enable => false} + }; +values(_) -> + #{}. %% ------------------------------------------------------------------------------------------------- %% Hocon Schema Definitions @@ -50,41 +65,45 @@ namespace() -> "bridge_hstreamdb". roots() -> []. fields("config") -> - [ - {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})}, - {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, - {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, - {payload, mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("payload")})}, - {connector, field(connector)} - ]; + hstream_bridge_common_fields() ++ + connector_fields(); fields("post") -> - [type_field(), name_field() | fields("config")]; -fields("put") -> - fields("config"); + hstream_bridge_common_fields() ++ + connector_fields() ++ + type_name_fields(); fields("get") -> - emqx_bridge_schema:status_fields() ++ fields("post"). + hstream_bridge_common_fields() ++ + connector_fields() ++ + type_name_fields() ++ + emqx_bridge_schema:status_fields(); +fields("put") -> + hstream_bridge_common_fields() ++ + connector_fields(). -field(connector) -> - mk( - hoconsc:union([binary(), ref(emqx_bridge_hstreamdb_connector, config)]), - #{ - required => true, - example => <<"hstreamdb:demo">>, - desc => ?DESC("desc_connector") - } - ). +hstream_bridge_common_fields() -> + emqx_bridge_schema:common_bridge_fields() ++ + [ + {direction, mk(egress, #{desc => ?DESC("config_direction"), default => egress})}, + {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, + {record_template, + mk(binary(), #{default => <<"${payload}">>, desc => ?DESC("record_template")})} + ] ++ + emqx_resource_schema:fields("resource_opts"). + +connector_fields() -> + emqx_bridge_hstreamdb_connector:fields(config). desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> - ["Configuration for HStream using `", string:to_upper(Method), "` method."]; + ["Configuration for HStreamDB bridge using `", string:to_upper(Method), "` method."]; desc(_) -> undefined. %% ------------------------------------------------------------------------------------------------- %% internal -type_field() -> - {type, mk(enum([hstreamdb]), #{required => true, desc => ?DESC("desc_type")})}. - -name_field() -> - {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})}. +type_name_fields() -> + [ + {type, mk(enum([hstreamdb]), #{required => true, desc => ?DESC("desc_type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} + ]. diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index c2a210271..31627db81 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -6,6 +6,7 @@ -include_lib("hocon/include/hoconsc.hrl"). -include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). -import(hoconsc, [mk/2, enum/1]). @@ -17,6 +18,7 @@ on_start/2, on_stop/2, on_query/3, + on_batch_query/3, on_get_status/2 ]). @@ -28,10 +30,12 @@ namespace/0, roots/0, fields/1, - desc/1, - connector_examples/1 + desc/1 ]). +-define(DEFAULT_GRPC_TIMEOUT, timer:seconds(30)). +-define(DEFAULT_GRPC_TIMEOUT_RAW, <<"30s">>). + %% ------------------------------------------------------------------------------------------------- %% resource callback callback_mode() -> always_sync. @@ -51,13 +55,35 @@ on_stop(InstId, #{client := Client, producer := Producer}) -> stop_producer => StopProducerRes }). +-define(FAILED_TO_APPLY_HRECORD_TEMPLATE, + {error, {unrecoverable_error, failed_to_apply_hrecord_template}} +). + on_query( _InstId, {send_message, Data}, - #{producer := Producer, ordering_key := OrderingKey, payload := Payload} + _State = #{ + producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate + } ) -> - Record = to_record(OrderingKey, Payload, Data), - do_append(Producer, Record). + try to_record(PartitionKey, HRecordTemplate, Data) of + Record -> append_record(Producer, Record) + catch + _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE + end. + +on_batch_query( + _InstId, + BatchList, + _State = #{ + producer := Producer, partition_key := PartitionKey, record_template := HRecordTemplate + } +) -> + try to_multi_part_records(PartitionKey, HRecordTemplate, BatchList) of + Records -> append_record(Producer, Records) + catch + _:_ -> ?FAILED_TO_APPLY_HRECORD_TEMPLATE + end. on_get_status(_InstId, #{client := Client}) -> case is_alive(Client) of @@ -87,43 +113,16 @@ fields(config) -> [ {url, mk(binary(), #{required => true, desc => ?DESC("url")})}, {stream, mk(binary(), #{required => true, desc => ?DESC("stream_name")})}, - {ordering_key, mk(binary(), #{required => false, desc => ?DESC("ordering_key")})}, - {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})} - ]; -fields("get") -> - fields("post"); -fields("put") -> - fields(config); -fields("post") -> - [ - {type, mk(hstreamdb, #{required => true, desc => ?DESC("type")})}, - {name, mk(binary(), #{required => true, desc => ?DESC("name")})} - ] ++ fields("put"). + {partition_key, mk(binary(), #{required => false, desc => ?DESC("partition_key")})}, + {pool_size, mk(pos_integer(), #{required => true, desc => ?DESC("pool_size")})}, + {grpc_timeout, fun grpc_timeout/1} + ] ++ emqx_connector_schema_lib:ssl_fields(). -connector_examples(Method) -> - [ - #{ - <<"hstreamdb">> => #{ - summary => <<"HStreamDB Connector">>, - value => values(Method) - } - } - ]. - -values(post) -> - maps:merge(values(put), #{name => <<"connector">>}); -values(get) -> - values(post); -values(put) -> - #{ - type => hstreamdb, - url => <<"http://127.0.0.1:6570">>, - stream => <<"stream1">>, - ordering_key => <<"some_key">>, - pool_size => 8 - }; -values(_) -> - #{}. +grpc_timeout(type) -> emqx_schema:timeout_duration_ms(); +grpc_timeout(desc) -> ?DESC("grpc_timeout"); +grpc_timeout(default) -> ?DEFAULT_GRPC_TIMEOUT_RAW; +grpc_timeout(required) -> false; +grpc_timeout(_) -> undefined. desc(config) -> ?DESC("config"). @@ -168,6 +167,10 @@ do_start_client(InstId, Config = #{url := Server, pool_size := PoolSize}) -> }), start_producer(InstId, Client, Config); _ -> + ?tp( + hstreamdb_connector_start_failed, + #{error => client_not_alive} + ), ?SLOG(error, #{ msg => "hstreamdb connector: client not alive", connector => InstId @@ -202,7 +205,7 @@ is_alive(Client) -> start_producer( InstId, Client, - Options = #{stream := Stream, pool_size := PoolSize, egress := #{payload := PayloadBin}} + Options = #{stream := Stream, pool_size := PoolSize} ) -> %% TODO: change these batch options after we have better disk cache. BatchSize = maps:get(batch_size, Options, 100), @@ -212,7 +215,8 @@ start_producer( {callback, {?MODULE, on_flush_result, []}}, {max_records, BatchSize}, {interval, Interval}, - {pool_size, PoolSize} + {pool_size, PoolSize}, + {grpc_timeout, maps:get(grpc_timeout, Options, ?DEFAULT_GRPC_TIMEOUT)} ], Name = produce_name(InstId), ?SLOG(info, #{ @@ -224,16 +228,14 @@ start_producer( ?SLOG(info, #{ msg => "hstreamdb connector: producer started" }), - EnableBatch = maps:get(enable_batch, Options, false), - Payload = emqx_placeholder:preproc_tmpl(PayloadBin), - OrderingKeyBin = maps:get(ordering_key, Options, <<"">>), - OrderingKey = emqx_placeholder:preproc_tmpl(OrderingKeyBin), State = #{ client => Client, producer => Producer, - enable_batch => EnableBatch, - ordering_key => OrderingKey, - payload => Payload + enable_batch => maps:get(enable_batch, Options, false), + partition_key => emqx_placeholder:preproc_tmpl( + maps:get(partition_key, Options, <<"">>) + ), + record_template => record_template(Options) }, {ok, State}; {error, {already_started, Pid}} -> @@ -253,47 +255,53 @@ start_producer( {error, Reason} end. -to_record(OrderingKeyTmpl, PayloadTmpl, Data) -> - OrderingKey = emqx_placeholder:proc_tmpl(OrderingKeyTmpl, Data), - Payload = emqx_placeholder:proc_tmpl(PayloadTmpl, Data), - to_record(OrderingKey, Payload). +to_record(PartitionKeyTmpl, HRecordTmpl, Data) -> + PartitionKey = emqx_placeholder:proc_tmpl(PartitionKeyTmpl, Data), + RawRecord = emqx_placeholder:proc_tmpl(HRecordTmpl, Data), + to_record(PartitionKey, RawRecord). -to_record(OrderingKey, Payload) when is_binary(OrderingKey) -> - to_record(binary_to_list(OrderingKey), Payload); -to_record(OrderingKey, Payload) -> - hstreamdb:to_record(OrderingKey, raw, Payload). +to_record(PartitionKey, RawRecord) when is_binary(PartitionKey) -> + to_record(binary_to_list(PartitionKey), RawRecord); +to_record(PartitionKey, RawRecord) -> + hstreamdb:to_record(PartitionKey, raw, RawRecord). -do_append(Producer, Record) -> - do_append(false, Producer, Record). +to_multi_part_records(PartitionKeyTmpl, HRecordTmpl, BatchList) -> + Records0 = lists:map( + fun({send_message, Data}) -> + to_record(PartitionKeyTmpl, HRecordTmpl, Data) + end, + BatchList + ), + PartitionKeys = proplists:get_keys(Records0), + [ + {PartitionKey, proplists:get_all_values(PartitionKey, Records0)} + || PartitionKey <- PartitionKeys + ]. -%% TODO: this append is async, remove or change it after we have better disk cache. -% do_append(true, Producer, Record) -> -% case hstreamdb:append(Producer, Record) of -% ok -> -% ?SLOG(debug, #{ -% msg => "hstreamdb producer async append success", -% record => Record -% }); -% {error, Reason} = Err -> -% ?SLOG(error, #{ -% msg => "hstreamdb producer async append failed", -% reason => Reason, -% record => Record -% }), -% Err -% end; -do_append(false, Producer, Record) -> - %% TODO: this append is sync, but it does not support [Record], can only append one Record. - %% Change it after we have better dick cache. +append_record(Producer, MultiPartsRecords) when is_list(MultiPartsRecords) -> + lists:foreach(fun(Record) -> append_record(Producer, Record) end, MultiPartsRecords); +append_record(Producer, Record) when is_tuple(Record) -> + do_append_records(false, Producer, Record). + +%% TODO: only sync request supported. implement async request later. +do_append_records(false, Producer, Record) -> case hstreamdb:append_flush(Producer, Record) of - {ok, _} -> + {ok, _Result} -> + ?tp( + hstreamdb_connector_query_return, + #{result => _Result} + ), ?SLOG(debug, #{ - msg => "hstreamdb producer sync append success", + msg => "HStreamDB producer sync append success", record => Record }); {error, Reason} = Err -> + ?tp( + hstreamdb_connector_query_return, + #{error => Reason} + ), ?SLOG(error, #{ - msg => "hstreamdb producer sync append failed", + msg => "HStreamDB producer sync append failed", reason => Reason, record => Record }), @@ -306,6 +314,11 @@ client_name(InstId) -> produce_name(ActionId) -> list_to_atom("producer:" ++ to_string(ActionId)). +record_template(#{record_template := RawHRecordTemplate}) -> + emqx_placeholder:preproc_tmpl(RawHRecordTemplate); +record_template(_) -> + emqx_placeholder:preproc_tmpl(<<"${payload}">>). + to_string(List) when is_list(List) -> List; to_string(Bin) when is_binary(Bin) -> binary_to_list(Bin); to_string(Atom) when is_atom(Atom) -> atom_to_list(Atom). diff --git a/changes/ee/feat-10203.en.md b/changes/ee/feat-10203.en.md new file mode 100644 index 000000000..a2ff3b3bb --- /dev/null +++ b/changes/ee/feat-10203.en.md @@ -0,0 +1 @@ +Add HStreamDB bridge support, adapted to the HStreamDB `v0.15.0`. diff --git a/rel/i18n/emqx_bridge_hstreamdb.hocon b/rel/i18n/emqx_bridge_hstreamdb.hocon index d9e7f1561..10700d4eb 100644 --- a/rel/i18n/emqx_bridge_hstreamdb.hocon +++ b/rel/i18n/emqx_bridge_hstreamdb.hocon @@ -6,12 +6,6 @@ config_direction.desc: config_direction.label: """Bridge Direction""" -config_enable.desc: -"""Enable or disable this bridge""" - -config_enable.label: -"""Enable Or Disable Bridge""" - desc_config.desc: """Configuration for an HStreamDB bridge.""" @@ -46,10 +40,10 @@ will be forwarded.""" local_topic.label: """Local Topic""" -payload.desc: -"""The payload to be forwarded to the HStreamDB. Placeholders supported.""" +record_template.desc: +"""The HStream Record template to be forwarded to the HStreamDB. Placeholders supported.""" -payload.label: -"""Payload""" +record_template.label: +"""HStream Record""" } diff --git a/rel/i18n/emqx_bridge_hstreamdb_connector.hocon b/rel/i18n/emqx_bridge_hstreamdb_connector.hocon index 001340e9c..c0faa794c 100644 --- a/rel/i18n/emqx_bridge_hstreamdb_connector.hocon +++ b/rel/i18n/emqx_bridge_hstreamdb_connector.hocon @@ -6,16 +6,34 @@ config.desc: config.label: """Connection config""" +type.desc: +"""The Connector Type.""" + +type.label: +"""Connector Type""" + name.desc: """Connector name, used as a human-readable description of the connector.""" name.label: """Connector Name""" -ordering_key.desc: +url.desc: +"""HStreamDB Server URL""" + +url.label: +"""HStreamDB Server URL""" + +stream_name.desc: +"""HStreamDB Stream Name""" + +stream_name.label: +"""HStreamDB Stream Name""" + +partition_key.desc: """HStreamDB Ordering Key""" -ordering_key.label: +partition_key.label: """HStreamDB Ordering Key""" pool_size.desc: @@ -24,22 +42,10 @@ pool_size.desc: pool_size.label: """HStreamDB Pool Size""" -stream_name.desc: -"""HStreamDB Stream Name""" +grpc_timeout.desc: +"""HStreamDB gRPC Timeout""" -stream_name.label: -"""HStreamDB Stream Name""" - -type.desc: -"""The Connector Type.""" - -type.label: -"""Connector Type""" - -url.desc: -"""HStreamDB Server URL""" - -url.label: -"""HStreamDB Server URL""" +grpc_timeout.label: +"""HStreamDB gRPC Timeout""" } diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 03587aa54..953b0b762 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -270,6 +270,10 @@ hstream hstreamDB hstream hstreamdb +hrecord +hRecord +Hrecord +HRecord SASL GSSAPI keytab From 583ccfaafd1375046203e38fd9836556db25b4b6 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Tue, 4 Jul 2023 18:44:07 +0800 Subject: [PATCH 39/92] test(hstreamdb): create stream and wirte data --- .../docker-compose-hstreamdb.yaml | 123 ++++ .../docker-compose-toxiproxy.yaml | 4 +- .ci/docker-compose-file/toxiproxy.json | 6 + apps/emqx_bridge_hstreamdb/docker-ct | 2 + .../include/emqx_bridge_hstreamdb.hrl | 5 + .../test/emqx_bridge_hstreamdb_SUITE.erl | 567 +++++++++++++++++- .../test/emqx_connector_hstreamdb_SUITE.erl | 16 - scripts/ct/run.sh | 3 + 8 files changed, 707 insertions(+), 19 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-hstreamdb.yaml create mode 100644 apps/emqx_bridge_hstreamdb/docker-ct create mode 100644 apps/emqx_bridge_hstreamdb/include/emqx_bridge_hstreamdb.hrl delete mode 100644 apps/emqx_bridge_hstreamdb/test/emqx_connector_hstreamdb_SUITE.erl diff --git a/.ci/docker-compose-file/docker-compose-hstreamdb.yaml b/.ci/docker-compose-file/docker-compose-hstreamdb.yaml new file mode 100644 index 000000000..f3c4dbd4c --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-hstreamdb.yaml @@ -0,0 +1,123 @@ +version: "3.5" + +services: + hserver: + image: hstreamdb/hstream:v0.15.0 + container_name: hstreamdb + depends_on: + - zookeeper + - hstore + # ports: + # - "127.0.0.1:6570:6570" + expose: + - 6570 + networks: + - emqx_bridge + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /tmp:/tmp + - data_store:/data/store + command: + - bash + - "-c" + - | + set -e + /usr/local/script/wait-for-storage.sh hstore 6440 zookeeper 2181 600 \ + /usr/local/bin/hstream-server \ + --bind-address 0.0.0.0 --port 6570 \ + --internal-port 6571 \ + --server-id 100 \ + --seed-nodes "$$(hostname -I | awk '{print $$1}'):6571" \ + --advertised-address $$(hostname -I | awk '{print $$1}') \ + --metastore-uri zk://zookeeper:2181 \ + --store-config /data/store/logdevice.conf \ + --store-admin-host hstore --store-admin-port 6440 \ + --store-log-level warning \ + --io-tasks-path /tmp/io/tasks \ + --io-tasks-network emqx_bridge + + hstore: + image: hstreamdb/hstream:v0.15.0 + networks: + - emqx_bridge + volumes: + - data_store:/data/store + command: + - bash + - "-c" + - | + set -ex + # N.B. "enable-dscp-reflection=false" is required for linux kernel which + # doesn't support dscp reflection, e.g. centos7. + /usr/local/bin/ld-dev-cluster --root /data/store \ + --use-tcp --tcp-host $$(hostname -I | awk '{print $$1}') \ + --user-admin-port 6440 \ + --param enable-dscp-reflection=false \ + --no-interactive + + zookeeper: + image: zookeeper + expose: + - 2181 + networks: + - emqx_bridge + volumes: + - data_zk_data:/data + - data_zk_datalog:/datalog + + ## The three container `hstream-exporter`, `prometheus`, `console` + ## is for HStreamDB Web Console + ## But HStreamDB Console is not supported in v0.15.0 + ## because of HStreamApi proto changed + # hstream-exporter: + # depends_on: + # hserver: + # condition: service_completed_successfully + # image: hstreamdb/hstream-exporter + # networks: + # - hstream-quickstart + # command: + # - bash + # - "-c" + # - | + # set -ex + # hstream-exporter --addr hstream://hserver:6570 + + # prometheus: + # image: prom/prometheus + # expose: + # - 9097 + # networks: + # - hstream-quickstart + # ports: + # - "9097:9090" + # volumes: + # - $PWD/prometheus:/etc/prometheus + + # console: + # image: hstreamdb/hstream-console + # depends_on: + # - hserver + # expose: + # - 5177 + # networks: + # - hstream-quickstart + # environment: + # - SERVER_PORT=5177 + # - PROMETHEUS_URL=http://prometheus:9097 + # - HSTREAM_PUBLIC_ADDRESS=hstream.example.com + # - HSTREAM_PRIVATE_ADDRESS=hserver:6570 + # ports: + # - "5177:5177" + +# networks: +# hstream-quickstart: +# name: hstream-quickstart + +volumes: + data_store: + name: quickstart_data_store + data_zk_data: + name: quickstart_data_zk_data + data_zk_datalog: + name: quickstart_data_zk_datalog diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index f15e779db..c0c88aef0 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -43,10 +43,12 @@ services: - 19000:19000 # S3 TLS - 19100:19100 - # IOTDB + # IOTDB (3 total) - 14242:4242 - 28080:18080 - 38080:38080 + # HStreamDB + - 15670:5670 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index 87878ac92..d5576108f 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -155,5 +155,11 @@ "listen": "0.0.0.0:8085", "upstream": "gcp_emulator:8085", "enabled": true + }, + { + "name": "hstreamdb", + "listen": "0.0.0.0:6570", + "upstream": "hstreamdb:6570", + "enabled": true } ] diff --git a/apps/emqx_bridge_hstreamdb/docker-ct b/apps/emqx_bridge_hstreamdb/docker-ct new file mode 100644 index 000000000..d25a92b6b --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +hstreamdb diff --git a/apps/emqx_bridge_hstreamdb/include/emqx_bridge_hstreamdb.hrl b/apps/emqx_bridge_hstreamdb/include/emqx_bridge_hstreamdb.hrl new file mode 100644 index 000000000..6b99c507a --- /dev/null +++ b/apps/emqx_bridge_hstreamdb/include/emqx_bridge_hstreamdb.hrl @@ -0,0 +1,5 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-define(HSTREAMDB_DEFAULT_PORT, 6570). diff --git a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl index 4b12beed7..015bccbef 100644 --- a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +++ b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl @@ -7,10 +7,573 @@ -compile(nowarn_export_all). -compile(export_all). +-include_lib("emqx_bridge_hstreamdb.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +% SQL definitions +-define(STREAM, "stream"). +-define(REPLICATION_FACTOR, 1). +%% in seconds +-define(BACKLOG_RETENTION_SECOND, (24 * 60 * 60)). +-define(SHARD_COUNT, 1). + +-define(BRIDGE_NAME, <<"hstreamdb_demo_bridge">>). +-define(RECORD_TEMPLATE, + "{ \"temperature\": ${payload.temperature}, \"humidity\": ${payload.humidity} }" +). + +-define(POOL_SIZE, 8). +-define(BATCH_SIZE, 10). +-define(GRPC_TIMEOUT, "1s"). + +-define(WORKER_POOL_SIZE, 4). + +-define(WITH_CLIENT(Process), + Client = connect_direct_hstream(_Name = test_c, Config), + Process, + ok = disconnect(Client) +). + +%% How to run it locally (all commands are run in $PROJ_ROOT dir): +%% A: run ct on host +%% 1. Start all deps services +%% ```bash +%% sudo docker compose -f .ci/docker-compose-file/docker-compose.yaml \ +%% -f .ci/docker-compose-file/docker-compose-hstreamdb.yaml \ +%% -f .ci/docker-compose-file/docker-compose-toxiproxy.yaml \ +%% up --build +%% ``` +%% +%% 2. Run use cases with special environment variables +%% 6570 is toxiproxy exported port. +%% Local: +%% ```bash +%% HSTREAMDB_HOST=$REAL_TOXIPROXY_IP HSTREAMDB_PORT=6570 \ +%% PROXY_HOST=$REAL_TOXIPROXY_IP PROXY_PORT=6570 \ +%% ./rebar3 as test ct -c -v --readable true --name ct@127.0.0.1 \ +%% --suite apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +%% ``` +%% +%% B: run ct in docker container +%% run script: +%% ```bash +%% ./scripts/ct/run.sh --ci --app apps/emqx_bridge_hstreamdb/ -- \ +%% --name 'test@127.0.0.1' -c -v --readable true \ +%% --suite apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +%% ```` + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ all() -> - emqx_common_test_helpers:all(?MODULE). + [ + {group, sync} + ]. -%% TODO: +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + NonBatchCases = [t_write_timeout], + BatchingGroups = [{group, with_batch}, {group, without_batch}], + [ + {sync, BatchingGroups}, + {with_batch, TCs -- NonBatchCases}, + {without_batch, TCs} + ]. + +init_per_group(sync, Config) -> + [{query_mode, sync} | Config]; +init_per_group(with_batch, Config0) -> + Config = [{enable_batch, true} | Config0], + common_init(Config); +init_per_group(without_batch, Config0) -> + Config = [{enable_batch, false} | Config0], + common_init(Config); +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when Group =:= with_batch; Group =:= without_batch -> + connect_and_delete_stream(Config), + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok. + +init_per_testcase(t_to_hrecord_failed, Config) -> + meck:new([hstreamdb], [passthrough, no_history, no_link]), + meck:expect(hstreamdb, to_record, fun(_, _, _) -> error(trans_to_hrecord_failed) end), + Config; +init_per_testcase(_Testcase, Config) -> + %% drop stream and will create a new one in common_init/1 + %% TODO: create a new stream for each test case + delete_bridge(Config), + snabbkaffe:start_trace(), + Config. + +end_per_testcase(t_to_hrecord_failed, _Config) -> + meck:unload([hstreamdb]); +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok = snabbkaffe:stop(), + delete_bridge(Config), + ok. + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_setup_via_config_and_publish(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Data = rand_data(), + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, Data)), + #{?snk_kind := hstreamdb_connector_query_return}, + 10_000 + ), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(hstreamdb_connector_query_return, Trace0), + lists:foreach( + fun(EachTrace) -> + ?assertMatch(#{result := #{streamName := <>}}, EachTrace) + end, + Trace + ), + ok + end + ), + ok. + +t_setup_via_http_api_and_publish(Config) -> + BridgeType = ?config(hstreamdb_bridge_type, Config), + Name = ?config(hstreamdb_name, Config), + HStreamDBConfig0 = ?config(hstreamdb_config, Config), + HStreamDBConfig = HStreamDBConfig0#{ + <<"name">> => Name, + <<"type">> => BridgeType + }, + ?assertMatch( + {ok, _}, + create_bridge_http(HStreamDBConfig) + ), + Data = rand_data(), + ?check_trace( + begin + ?wait_async_action( + ?assertEqual(ok, send_message(Config, Data)), + #{?snk_kind := hstreamdb_connector_query_return}, + 10_000 + ), + ok + end, + fun(Trace) -> + ?assertMatch( + [#{result := #{streamName := <>}}], + ?of_kind(hstreamdb_connector_query_return, Trace) + ) + end + ), + ok. + +t_get_status(Config) -> + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + health_check_resource_ok(Config), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + health_check_resource_down(Config) + end), + ok. + +t_create_disconnected(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({ok, _}, create_bridge(Config)) + end), + fun(Trace) -> + ?assertMatch( + [#{error := client_not_alive}], + ?of_kind(hstreamdb_connector_start_failed, Trace) + ), + ok + end + ), + %% TODO: Investigate why reconnection takes at least 5 seconds during ct. + %% While in practical applications, recovers to the 'connected' state + %% within 3 seconds after toxiproxy being enabled.'" + %% timer:sleep(10000), + restart_resource(Config), + health_check_resource_ok(Config), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + QueryMode = ?config(query_mode, Config), + Data = rand_data(), + {{ok, _}, {ok, _}} = + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := resource_connected_enter}, + 20_000 + ), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + health_check_resource_down(Config), + case QueryMode of + sync -> + ?assertMatch( + {error, {resource_error, #{msg := "call resource timeout", reason := timeout}}}, + send_message(Config, Data) + ); + async -> + %% TODO: async mode is not supported yet, + %% but it will return ok if calling emqx_resource_buffer_worker:async_query/3, + ?assertMatch( + ok, + send_message(Config, Data) + ) + end + end), + ok. + +t_simple_query(Config) -> + BatchSize = batch_size(Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Requests = gen_batch_req(BatchSize), + ?check_trace( + begin + ?wait_async_action( + lists:foreach( + fun(Request) -> + ?assertEqual(ok, query_resource(Config, Request)) + end, + Requests + ), + #{?snk_kind := hstreamdb_connector_query_return}, + 10_000 + ) + end, + fun(Trace0) -> + Trace = ?of_kind(hstreamdb_connector_query_return, Trace0), + lists:foreach( + fun(EachTrace) -> + ?assertMatch(#{result := #{streamName := <>}}, EachTrace) + end, + Trace + ), + ok + end + ), + ok. + +t_to_hrecord_failed(Config) -> + QueryMode = ?config(query_mode, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + Result = send_message(Config, #{}), + case QueryMode of + sync -> + ?assertMatch( + {error, {unrecoverable_error, failed_to_apply_hrecord_template}}, + Result + ) + %% TODO: async mode is not supported yet + end, + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +common_init(ConfigT) -> + Host = os:getenv("HSTREAMDB_HOST", "toxiproxy"), + RawPort = os:getenv("HSTREAMDB_PORT", str(?HSTREAMDB_DEFAULT_PORT)), + Port = list_to_integer(RawPort), + URL = "http://" ++ Host ++ ":" ++ RawPort, + + Config0 = [ + {hstreamdb_host, Host}, + {hstreamdb_port, Port}, + {hstreamdb_url, URL}, + %% see also for `proxy_name` : $PROJ_ROOT/.ci/docker-compose-file/toxiproxy.json + {proxy_name, "hstreamdb"}, + {batch_size, batch_size(ConfigT)} + | ConfigT + ], + + BridgeType = proplists:get_value(bridge_type, Config0, <<"hstreamdb">>), + case emqx_common_test_helpers:is_tcp_server_available(Host, Port) of + true -> + % Setup toxiproxy + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + % Ensure EE bridge module is loaded + _ = application:load(emqx_ee_bridge), + _ = application:ensure_all_started(hstreamdb_erl), + _ = emqx_ee_bridge:module_info(), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + emqx_mgmt_api_test_util:init_suite(), + % Connect to hstreamdb directly + % drop old stream and then create new one + connect_and_delete_stream(Config0), + connect_and_create_stream(Config0), + {Name, HStreamDBConf} = hstreamdb_config(BridgeType, Config0), + Config = + [ + {hstreamdb_config, HStreamDBConf}, + {hstreamdb_bridge_type, BridgeType}, + {hstreamdb_name, Name}, + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort} + | Config0 + ], + Config; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_hstreamdb); + _ -> + {skip, no_hstreamdb} + end + end. + +hstreamdb_config(BridgeType, Config) -> + Port = integer_to_list(?config(hstreamdb_port, Config)), + URL = "http://" ++ ?config(hstreamdb_host, Config) ++ ":" ++ Port, + Name = ?BRIDGE_NAME, + BatchSize = batch_size(Config), + ConfigString = + io_lib:format( + "bridges.~s.~s {\n" + " enable = true\n" + " url = ~p\n" + " stream = ~p\n" + " record_template = ~p\n" + " pool_size = ~p\n" + " grpc_timeout = ~p\n" + " resource_opts = {\n" + %% always sync + " query_mode = sync\n" + " request_ttl = 500ms\n" + " batch_size = ~b\n" + " worker_pool_size = ~b\n" + " }\n" + "}", + [ + BridgeType, + Name, + URL, + ?STREAM, + ?RECORD_TEMPLATE, + ?POOL_SIZE, + ?GRPC_TIMEOUT, + BatchSize, + ?WORKER_POOL_SIZE + ] + ), + {Name, parse_and_check(ConfigString, BridgeType, Name)}. + +parse_and_check(ConfigString, BridgeType, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{BridgeType := #{Name := Config}}} = RawConf, + Config. + +-define(RPC_OPTIONS, #{pool_size => 4}). + +-define(CONN_ATTEMPTS, 10). + +default_options(Config) -> + [ + {url, ?config(hstreamdb_url, Config)}, + {rpc_options, ?RPC_OPTIONS} + ]. + +connect_direct_hstream(Name, Config) -> + client(Name, Config, ?CONN_ATTEMPTS). + +client(_Name, _Config, N) when N =< 0 -> error(cannot_connect); +client(Name, Config, N) -> + try + _ = hstreamdb:stop_client(Name), + {ok, Client} = hstreamdb:start_client(Name, default_options(Config)), + {ok, echo} = hstreamdb:echo(Client), + Client + catch + Class:Error -> + ct:print("Error connecting: ~p", [{Class, Error}]), + ct:sleep(timer:seconds(1)), + client(Name, Config, N - 1) + end. + +disconnect(Client) -> + hstreamdb:stop_client(Client). + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + BridgeType = ?config(hstreamdb_bridge_type, Config), + Name = ?config(hstreamdb_name, Config), + HSDBConfig0 = ?config(hstreamdb_config, Config), + HSDBConfig = emqx_utils_maps:deep_merge(HSDBConfig0, Overrides), + emqx_bridge:create(BridgeType, Name, HSDBConfig). + +delete_bridge(Config) -> + BridgeType = ?config(hstreamdb_bridge_type, Config), + Name = ?config(hstreamdb_name, Config), + emqx_bridge:remove(BridgeType, Name). + +create_bridge_http(Params) -> + Path = emqx_mgmt_api_test_util:api_path(["bridges"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Data) -> + Name = ?config(hstreamdb_name, Config), + BridgeType = ?config(hstreamdb_bridge_type, Config), + BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name), + emqx_bridge:send_message(BridgeID, Data). + +query_resource(Config, Request) -> + Name = ?config(hstreamdb_name, Config), + BridgeType = ?config(hstreamdb_bridge_type, Config), + ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name), + emqx_resource:query(ResourceID, Request, #{timeout => 1_000}). + +restart_resource(Config) -> + BridgeName = ?config(hstreamdb_name, Config), + BridgeType = ?config(hstreamdb_bridge_type, Config), + emqx_bridge:disable_enable(disable, BridgeType, BridgeName), + timer:sleep(200), + emqx_bridge:disable_enable(enable, BridgeType, BridgeName). + +resource_id(Config) -> + BridgeName = ?config(hstreamdb_name, Config), + BridgeType = ?config(hstreamdb_bridge_type, Config), + _ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName). + +health_check_resource_ok(Config) -> + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(resource_id(Config))). + +health_check_resource_down(Config) -> + case emqx_resource_manager:health_check(resource_id(Config)) of + {ok, Status} when Status =:= disconnected orelse Status =:= connecting -> + ok; + {error, timeout} -> + ok; + Other -> + ?assert( + false, lists:flatten(io_lib:format("invalid health check result:~p~n", [Other])) + ) + end. + +% These funs start and then stop the hstreamdb connection +connect_and_create_stream(Config) -> + ?WITH_CLIENT( + _ = hstreamdb:create_stream( + Client, ?STREAM, ?REPLICATION_FACTOR, ?BACKLOG_RETENTION_SECOND, ?SHARD_COUNT + ) + ), + %% force write to stream to make it created and ready to be written data for rest cases + ProducerOptions = [ + {pool_size, 4}, + {stream, ?STREAM}, + {callback, fun(_) -> ok end}, + {max_records, 10}, + {interval, 1000} + ], + ?WITH_CLIENT( + begin + {ok, Producer} = hstreamdb:start_producer(Client, test_producer, ProducerOptions), + _ = hstreamdb:append_flush(Producer, hstreamdb:to_record([], raw, rand_payload())), + _ = hstreamdb:stop_producer(Producer) + end + ). + +connect_and_delete_stream(Config) -> + ?WITH_CLIENT( + _ = hstreamdb:delete_stream(Client, ?STREAM) + ). + +%%-------------------------------------------------------------------- +%% help functions +%%-------------------------------------------------------------------- + +batch_size(Config) -> + case ?config(enable_batch, Config) of + true -> ?BATCH_SIZE; + false -> 1 + end. + +rand_data() -> + #{ + %% Raw MTTT Payload in binary + payload => rand_payload(), + id => <<"0005F8F84FFFAFB9F44200000D810002">>, + topic => <<"test/topic">>, + qos => 0 + }. + +rand_payload() -> + emqx_utils_json:encode(#{ + temperature => rand:uniform(40), humidity => rand:uniform(100) + }). + +gen_batch_req(Count) when + is_integer(Count) andalso Count > 0 +-> + [{send_message, rand_data()} || _Val <- lists:seq(1, Count)]; +gen_batch_req(Count) -> + ct:pal("Gen batch requests failed with unexpected Count: ~p", [Count]). + +str(List) when is_list(List) -> + unicode:characters_to_list(List, utf8); +str(Bin) when is_binary(Bin) -> + unicode:characters_to_list(Bin, utf8); +str(Num) when is_number(Num) -> + number_to_list(Num). + +number_to_list(Int) when is_integer(Int) -> + integer_to_list(Int); +number_to_list(Float) when is_float(Float) -> + float_to_list(Float, [{decimals, 10}, compact]). diff --git a/apps/emqx_bridge_hstreamdb/test/emqx_connector_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_connector_hstreamdb_SUITE.erl deleted file mode 100644 index 09ba487f7..000000000 --- a/apps/emqx_bridge_hstreamdb/test/emqx_connector_hstreamdb_SUITE.erl +++ /dev/null @@ -1,16 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. -%%-------------------------------------------------------------------- - --module(emqx_connector_hstreamdb_SUITE). - --compile(nowarn_export_all). --compile(export_all). - --include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). - -all() -> - emqx_common_test_helpers:all(?MODULE). - -%% TODO: diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index ea49f6e21..e4061f7cb 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -216,6 +216,9 @@ for dep in ${CT_DEPS}; do gcp_emulator) FILES+=( '.ci/docker-compose-file/docker-compose-gcp-emulator.yaml' ) ;; + hstreamdb) + FILES+=( '.ci/docker-compose-file/docker-compose-hstreamdb.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From b089fba100d8d0d422d67e2c98d3d7373c1fcc1f Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 6 Jul 2023 11:19:28 +0800 Subject: [PATCH 40/92] refactor: rm ee_bridge and ee_connector application --- apps/emqx_bridge/src/emqx_bridge.app.src | 1 + apps/emqx_bridge/src/emqx_bridge_api.erl | 8 ++--- apps/emqx_bridge/src/emqx_bridge_app.erl | 8 ++--- apps/emqx_bridge/src/emqx_bridge_resource.erl | 2 +- .../src/schema/emqx_bridge_enterprise.erl | 8 ++++- .../src/schema/emqx_bridge_schema.erl | 33 ++++++------------- .../src/emqx_bridge_cassandra.app.src | 10 ++++-- .../src/emqx_bridge_cassandra_connector.erl | 2 +- .../test/emqx_bridge_cassandra_SUITE.erl | 3 +- .../emqx_bridge_cassandra_connector_SUITE.erl | 4 +-- .../src/emqx_bridge_clickhouse.app.src | 10 ++++-- .../src/emqx_bridge_clickhouse_connector.erl | 2 +- .../test/emqx_bridge_clickhouse_SUITE.erl | 2 +- .../src/emqx_bridge_dynamo.app.src | 10 ++++-- .../test/emqx_bridge_dynamo_SUITE.erl | 12 ++++--- .../src/emqx_bridge_gcp_pubsub.app.src | 4 ++- .../src/emqx_bridge_gcp_pubsub.erl | 2 +- .../src/emqx_bridge_hstreamdb.app.src | 7 +++- .../test/emqx_bridge_hstreamdb_SUITE.erl | 7 ++-- .../src/emqx_bridge_influxdb.app.src | 10 ++++-- .../src/emqx_bridge_iotdb.app.src | 5 ++- .../src/emqx_bridge_iotdb.erl | 2 +- .../src/emqx_bridge_kafka.app.src | 5 ++- .../src/emqx_bridge_kafka_impl_producer.erl | 2 +- .../emqx_bridge_kafka_impl_producer_SUITE.erl | 8 ++--- .../src/emqx_bridge_matrix.app.src | 9 +++-- .../src/emqx_bridge_mongodb.app.src | 3 +- .../src/emqx_bridge_mongodb.erl | 2 +- .../src/emqx_bridge_mongodb_connector.erl | 2 +- .../test/emqx_bridge_mongodb_SUITE.erl | 13 ++++---- .../src/emqx_bridge_mysql.app.src | 11 +++++-- .../test/emqx_bridge_mysql_SUITE.erl | 5 ++- .../src/emqx_bridge_opents.app.src | 4 ++- .../test/emqx_bridge_opents_SUITE.erl | 12 ++++--- .../src/emqx_bridge_oracle.app.src | 4 ++- .../test/emqx_bridge_oracle_SUITE.erl | 5 +-- .../src/emqx_bridge_pgsql.app.src | 9 +++-- .../test/emqx_bridge_pgsql_SUITE.erl | 5 ++- .../src/emqx_bridge_pulsar.app.src | 4 ++- .../src/emqx_bridge_pulsar.erl | 2 +- ...emqx_bridge_pulsar_impl_producer_SUITE.erl | 6 ++-- .../src/emqx_bridge_rabbitmq.app.src | 12 +++++-- .../test/emqx_bridge_rabbitmq_SUITE.erl | 5 +-- .../emqx_bridge_rabbitmq_connector_SUITE.erl | 1 - .../src/emqx_bridge_redis.app.src | 11 +++++-- .../src/emqx_bridge_redis_connector.erl | 16 ++++----- .../test/emqx_bridge_redis_SUITE.erl | 26 +++++++-------- .../src/emqx_bridge_rocketmq.app.src | 4 +-- .../test/emqx_bridge_rocketmq_SUITE.erl | 9 ++--- .../src/emqx_bridge_sqlserver.app.src | 4 +-- .../test/emqx_bridge_sqlserver_SUITE.erl | 7 ++-- .../src/emqx_bridge_tdengine.app.src | 10 ++++-- .../test/emqx_bridge_tdengine_SUITE.erl | 7 ++-- .../src/emqx_bridge_timescale.app.src | 4 +-- lib-ee/emqx_ee_bridge/.gitignore | 19 ----------- lib-ee/emqx_ee_bridge/README.md | 9 ----- lib-ee/emqx_ee_bridge/docker-ct | 1 - lib-ee/emqx_ee_bridge/rebar.config | 11 ------- .../emqx_ee_bridge/src/emqx_ee_bridge.app.src | 27 --------------- lib-ee/emqx_ee_connector/.gitignore | 19 ----------- lib-ee/emqx_ee_connector/README.md | 9 ----- lib-ee/emqx_ee_connector/docker-ct | 2 -- lib-ee/emqx_ee_connector/rebar.config | 10 ------ .../src/emqx_ee_connector.app.src | 16 --------- mix.exs | 2 -- rebar.config.erl | 2 -- 66 files changed, 222 insertions(+), 284 deletions(-) rename lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl => apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl (99%) delete mode 100644 lib-ee/emqx_ee_bridge/.gitignore delete mode 100644 lib-ee/emqx_ee_bridge/README.md delete mode 100644 lib-ee/emqx_ee_bridge/docker-ct delete mode 100644 lib-ee/emqx_ee_bridge/rebar.config delete mode 100644 lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src delete mode 100644 lib-ee/emqx_ee_connector/.gitignore delete mode 100644 lib-ee/emqx_ee_connector/README.md delete mode 100644 lib-ee/emqx_ee_connector/docker-ct delete mode 100644 lib-ee/emqx_ee_connector/rebar.config delete mode 100644 lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 07711da12..11d199c9d 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -8,6 +8,7 @@ kernel, stdlib, emqx, + emqx_resource, emqx_connector ]}, {env, []}, diff --git a/apps/emqx_bridge/src/emqx_bridge_api.erl b/apps/emqx_bridge/src/emqx_bridge_api.erl index 57933029d..a71315a27 100644 --- a/apps/emqx_bridge/src/emqx_bridge_api.erl +++ b/apps/emqx_bridge/src/emqx_bridge_api.erl @@ -175,14 +175,14 @@ bridge_info_examples(Method) -> value => info_example(mqtt, Method) } }, - ee_bridge_examples(Method) + emqx_enterprise_bridge_examples(Method) ). -if(?EMQX_RELEASE_EDITION == ee). -ee_bridge_examples(Method) -> - emqx_ee_bridge:examples(Method). +emqx_enterprise_bridge_examples(Method) -> + emqx_bridge_enterprise:examples(Method). -else. -ee_bridge_examples(_Method) -> #{}. +emqx_enterprise_bridge_examples(_Method) -> #{}. -endif. info_example(Type, Method) -> diff --git a/apps/emqx_bridge/src/emqx_bridge_app.erl b/apps/emqx_bridge/src/emqx_bridge_app.erl index 59c94cef7..3bae55090 100644 --- a/apps/emqx_bridge/src/emqx_bridge_app.erl +++ b/apps/emqx_bridge/src/emqx_bridge_app.erl @@ -31,7 +31,7 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_bridge_sup:start_link(), - ok = start_ee_apps(), + ok = ensure_enterprise_schema_loaded(), ok = emqx_bridge:load(), ok = emqx_bridge:load_hook(), ok = emqx_config_handler:add_handler(?LEAF_NODE_HDLR_PATH, ?MODULE), @@ -46,11 +46,11 @@ stop(_State) -> ok. -if(?EMQX_RELEASE_EDITION == ee). -start_ee_apps() -> - {ok, _} = application:ensure_all_started(emqx_ee_bridge), +ensure_enterprise_schema_loaded() -> + _ = emqx_bridge_enterprise:module_info(), ok. -else. -start_ee_apps() -> +ensure_enterprise_schema_loaded() -> ok. -endif. diff --git a/apps/emqx_bridge/src/emqx_bridge_resource.erl b/apps/emqx_bridge/src/emqx_bridge_resource.erl index db8669f49..203a65072 100644 --- a/apps/emqx_bridge/src/emqx_bridge_resource.erl +++ b/apps/emqx_bridge/src/emqx_bridge_resource.erl @@ -64,7 +64,7 @@ bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector; bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector; bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http; bridge_to_resource_type(webhook) -> emqx_connector_http; -bridge_to_resource_type(BridgeType) -> emqx_ee_bridge:resource_type(BridgeType). +bridge_to_resource_type(BridgeType) -> emqx_bridge_enterprise:resource_type(BridgeType). -else. bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector; bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector; diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl similarity index 99% rename from lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl rename to apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index 17da77680..e76d1af37 100644 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -1,7 +1,9 @@ %%-------------------------------------------------------------------- %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. %%-------------------------------------------------------------------- --module(emqx_ee_bridge). +-module(emqx_bridge_enterprise). + +-if(?EMQX_RELEASE_EDITION == ee). -include_lib("hocon/include/hoconsc.hrl"). -import(hoconsc, [mk/2, enum/1, ref/2]). @@ -365,3 +367,7 @@ rabbitmq_structs() -> api_ref(Module, Type, Method) -> {Type, ref(Module, Method)}. + +-else. + +-endif. diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl index 03ae781ca..58be231e4 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_schema.erl @@ -57,7 +57,7 @@ api_schema(Method) -> {<<"mqtt">>, emqx_bridge_mqtt_schema} ] ], - EE = ee_api_schemas(Method), + EE = enterprise_api_schemas(Method), hoconsc:union(bridge_api_union(Broker ++ EE)). bridge_api_union(Refs) -> @@ -86,36 +86,23 @@ bridge_api_union(Refs) -> end. -if(?EMQX_RELEASE_EDITION == ee). -ee_api_schemas(Method) -> - ensure_loaded(emqx_ee_bridge, emqx_ee_bridge), - case erlang:function_exported(emqx_ee_bridge, api_schemas, 1) of - true -> emqx_ee_bridge:api_schemas(Method); +enterprise_api_schemas(Method) -> + case erlang:function_exported(emqx_bridge_enterprise, api_schemas, 1) of + true -> emqx_bridge_enterprise:api_schemas(Method); false -> [] end. -ee_fields_bridges() -> - ensure_loaded(emqx_ee_bridge, emqx_ee_bridge), - case erlang:function_exported(emqx_ee_bridge, fields, 1) of - true -> emqx_ee_bridge:fields(bridges); +enterprise_fields_bridges() -> + case erlang:function_exported(emqx_bridge_enterprise, fields, 1) of + true -> emqx_bridge_enterprise:fields(bridges); false -> [] end. -%% must ensure the app is loaded before checking if fn is defined. -ensure_loaded(App, Mod) -> - try - _ = application:load(App), - _ = Mod:module_info(), - ok - catch - _:_ -> - ok - end. - -else. -ee_api_schemas(_) -> []. +enterprise_api_schemas(_) -> []. -ee_fields_bridges() -> []. +enterprise_fields_bridges() -> []. -endif. @@ -191,7 +178,7 @@ fields(bridges) -> end } )} - ] ++ ee_fields_bridges(); + ] ++ enterprise_fields_bridges(); fields("metrics") -> [ {"dropped", mk(integer(), #{desc => ?DESC("metric_dropped")})}, diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src index ea3495e0f..f449588cc 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_cassandra, [ {description, "EMQX Enterprise Cassandra Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, ecql]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + ecql + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl index ad41329d2..2cbf0d6fe 100644 --- a/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl +++ b/apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl @@ -396,7 +396,7 @@ conn_opts([Opt | Opts], Acc) -> %% prepare %% XXX: hardcode -%% note: the `cql` param is passed by emqx_ee_bridge_cassa +%% note: the `cql` param is passed by emqx_bridge_cassandra parse_prepare_cql(#{cql := SQL}) -> parse_prepare_cql([{send_message, SQL}], #{}, #{}); parse_prepare_cql(_) -> diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl index fb16dd749..9df219296 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl @@ -170,9 +170,8 @@ common_init(Config0) -> ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to cassnadra directly and create the table catch connect_and_drop_table(Config0), diff --git a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl index 452db33a7..bceae1fd2 100644 --- a/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl +++ b/apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_connector_SUITE.erl @@ -56,7 +56,6 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), %% keyspace `mqtt` must be created in advance {ok, Conn} = ecql:connect([ @@ -79,8 +78,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_conf]), ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector), - _ = application:stop(emqx_ee_connector). + _ = application:stop(emqx_connector). init_per_testcase(_, Config) -> Config. diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src index 58a92fde4..cfb08f47b 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_clickhouse, [ {description, "EMQX Enterprise ClickHouse Bridge"}, - {vsn, "0.2.1"}, + {vsn, "0.2.2"}, {registered, []}, - {applications, [kernel, stdlib, clickhouse, emqx_resource]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + clickhouse + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl index d0164b57c..98c524913 100644 --- a/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl +++ b/apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl @@ -469,7 +469,7 @@ transform_and_log_clickhouse_result(ClickhouseErrorResult, ResourceID, SQL) -> reason => ClickhouseErrorResult }), case is_recoverable_error(ClickhouseErrorResult) of - %% TODO: The hackeny errors that the clickhouse library forwards are + %% TODO: The hackney errors that the clickhouse library forwards are %% very loosely defined. We should try to make sure that the following %% handles all error cases that we need to handle as recoverable_error true -> diff --git a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl index 787fb81ff..b1a560442 100644 --- a/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl +++ b/apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_SUITE.erl @@ -12,7 +12,7 @@ -include_lib("emqx_connector/include/emqx_connector.hrl"). %% See comment in -%% lib-ee/emqx_ee_connector/test/ee_bridge_clickhouse_connector_SUITE.erl for how to +%% apps/emqx_bridge_clickhouse/test/emqx_bridge_clickhouse_connector_SUITE.erl for how to %% run this without bringing up the whole CI infrastucture %%------------------------------------------------------------------------------ diff --git a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src index 0e202b714..824f5ee7b 100644 --- a/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src +++ b/apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_dynamo, [ {description, "EMQX Enterprise Dynamo Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, erlcloud]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + erlcloud + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl index ac2b59229..9490e6455 100644 --- a/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl +++ b/apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl @@ -88,7 +88,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_conf, erlcloud]), ok. init_per_testcase(TestCase, Config) -> @@ -128,10 +128,12 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, emqx_resource, emqx_bridge + ]), + _ = application:ensure_all_started(erlcloud), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % setup dynamo setup_dynamo(Config0), diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src index 85bbfdd8c..bf5510366 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_gcp_pubsub, [ {description, "EMQX Enterprise GCP Pub/Sub Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, ehttpc ]}, {env, []}, diff --git a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl index 890a3faed..8ef369068 100644 --- a/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl +++ b/apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub.erl @@ -21,7 +21,7 @@ service_account_json_converter/1 ]). -%% emqx_ee_bridge "unofficial" API +%% emqx_bridge_enterprise "unofficial" API -export([conn_bridge_examples/1]). -type service_account_json() :: map(). diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index 32fbc29ac..2b1e96b00 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -2,7 +2,12 @@ {description, "EMQX Enterprise HStreamDB Bridge"}, {vsn, "0.1.1"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl index 015bccbef..430343274 100644 --- a/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl +++ b/apps/emqx_bridge_hstreamdb/test/emqx_bridge_hstreamdb_SUITE.erl @@ -108,7 +108,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_resource, emqx_conf, hstreamdb_erl]), ok. init_per_testcase(t_to_hrecord_failed, Config) -> @@ -344,10 +344,9 @@ common_init(ConfigT) -> ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_resource, emqx_bridge]), _ = application:ensure_all_started(hstreamdb_erl), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to hstreamdb directly % drop old stream and then create new one diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src index 80b708582..71b95a40d 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_influxdb, [ {description, "EMQX Enterprise InfluxDB Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, influxdb]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + influxdb + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src index a3e4f1eb3..869656dbd 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge_iotdb, [ {description, "EMQX Enterprise Apache IoTDB Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {modules, [ emqx_bridge_iotdb, emqx_bridge_iotdb_impl @@ -10,6 +10,9 @@ {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, + %% for module emqx_connector_http emqx_connector ]}, {env, []}, diff --git a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl index 724c3f43a..629ac0885 100644 --- a/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl +++ b/apps/emqx_bridge_iotdb/src/emqx_bridge_iotdb.erl @@ -18,7 +18,7 @@ desc/1 ]). -%% emqx_ee_bridge "unofficial" API +%% emqx_bridge_enterprise "unofficial" API -export([conn_bridge_examples/1]). %%------------------------------------------------------------------------------------------------- diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src index 59c26717e..87c1841e5 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src @@ -1,10 +1,13 @@ +%% -*- mode: erlang -*- {application, emqx_bridge_kafka, [ {description, "EMQX Enterprise Kafka Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, [emqx_bridge_kafka_consumer_sup]}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, telemetry, wolff, brod, diff --git a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl index 4fa188c95..bcdeaf870 100644 --- a/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl +++ b/apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl @@ -40,7 +40,7 @@ query_mode(_) -> callback_mode() -> async_if_possible. -%% @doc Config schema is defined in emqx_ee_bridge_kafka. +%% @doc Config schema is defined in emqx_bridge_kafka. on_start(InstId, Config) -> #{ authentication := Auth, diff --git a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl index 38d58c1e7..95dec2db0 100644 --- a/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl @@ -73,11 +73,9 @@ wait_until_kafka_is_up(Attempts) -> end. init_per_suite(Config) -> - %% ensure loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - application:load(emqx_bridge), - ok = emqx_common_test_helpers:start_apps([emqx_conf]), + %% Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src index 7dfe7eae6..42129bfc7 100644 --- a/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src +++ b/apps/emqx_bridge_matrix/src/emqx_bridge_matrix.app.src @@ -1,8 +1,13 @@ {application, emqx_bridge_matrix, [ {description, "EMQX Enterprise MatrixDB Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src index b10c92aef..fa3ebd3c9 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src @@ -1,6 +1,6 @@ {application, emqx_bridge_mongodb, [ {description, "EMQX Enterprise MongoDB Bridge"}, - {vsn, "0.2.0"}, + {vsn, "0.2.1"}, {registered, []}, {applications, [ kernel, @@ -8,7 +8,6 @@ emqx_connector, emqx_resource, emqx_bridge, - emqx_ee_bridge, emqx_mongodb ]}, {env, []}, diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl index 72485815f..b108f654f 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl @@ -10,7 +10,7 @@ -behaviour(hocon_schema). -%% emqx_ee_bridge "callbacks" +%% emqx_bridge_enterprise "callbacks" -export([ conn_bridge_examples/1 ]). diff --git a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl index eb0a22e9c..8c004d829 100644 --- a/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl +++ b/apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb_connector.erl @@ -58,7 +58,7 @@ on_query(InstanceId, {send_message, Message0}, State) -> }, Message = render_message(PayloadTemplate, Message0), Res = emqx_mongodb:on_query(InstanceId, {send_message, Message}, NewConnectorState), - ?tp(mongo_ee_connector_on_query_return, #{result => Res}), + ?tp(mongo_bridge_connector_on_query_return, #{result => Res}), Res; on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) -> emqx_mongodb:on_query(InstanceId, Request, ConnectorState). diff --git a/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl b/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl index 89243bf8e..758124713 100644 --- a/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl +++ b/apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl @@ -116,7 +116,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf, emqx_rule_engine]), + ok = emqx_common_test_helpers:stop_apps([emqx_mongodb, emqx_bridge, emqx_rule_engine, emqx_conf]), ok. init_per_testcase(_Testcase, Config) -> @@ -146,9 +146,8 @@ start_apps() -> ]). ensure_loaded() -> - _ = application:load(emqx_ee_bridge), _ = application:load(emqtt), - _ = emqx_ee_bridge:module_info(), + _ = emqx_bridge_enterprise:module_info(), ok. mongo_type(Config) -> @@ -354,7 +353,7 @@ t_setup_via_config_and_publish(Config) -> {ok, {ok, _}} = ?wait_async_action( send_message(Config, #{key => Val}), - #{?snk_kind := mongo_ee_connector_on_query_return}, + #{?snk_kind := mongo_bridge_connector_on_query_return}, 5_000 ), ?assertMatch( @@ -379,7 +378,7 @@ t_setup_via_http_api_and_publish(Config) -> {ok, {ok, _}} = ?wait_async_action( send_message(Config, #{key => Val}), - #{?snk_kind := mongo_ee_connector_on_query_return}, + #{?snk_kind := mongo_bridge_connector_on_query_return}, 5_000 ), ?assertMatch( @@ -395,7 +394,7 @@ t_payload_template(Config) -> {ok, {ok, _}} = ?wait_async_action( send_message(Config, #{key => Val, clientid => ClientId}), - #{?snk_kind := mongo_ee_connector_on_query_return}, + #{?snk_kind := mongo_bridge_connector_on_query_return}, 5_000 ), ?assertMatch( @@ -421,7 +420,7 @@ t_collection_template(Config) -> clientid => ClientId, mycollectionvar => <<"mycol">> }), - #{?snk_kind := mongo_ee_connector_on_query_return}, + #{?snk_kind := mongo_bridge_connector_on_query_return}, 5_000 ), ?assertMatch( diff --git a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src index 2e6844712..2ecdd6a6a 100644 --- a/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src +++ b/apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src @@ -1,8 +1,15 @@ {application, emqx_bridge_mysql, [ {description, "EMQX Enterprise MySQL Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, emqx_mysql]}, + {applications, [ + kernel, + stdlib, + emqx_connector, + emqx_resource, + emqx_bridge, + emqx_mysql + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl index 2ca0f410d..3ed40e903 100644 --- a/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl +++ b/apps/emqx_bridge_mysql/test/emqx_bridge_mysql_SUITE.erl @@ -142,10 +142,9 @@ common_init(Config0) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), + % Ensure enterprise bridge module is loaded ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge, emqx_rule_engine]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to mysql directly and create the table connect_and_create_table(Config0), diff --git a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src index 9037b8840..6ec938afd 100644 --- a/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src +++ b/apps/emqx_bridge_opents/src/emqx_bridge_opents.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_opents, [ {description, "EMQX Enterprise OpenTSDB Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, opentsdb ]}, {env, []}, diff --git a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl index 93224d5ca..3632ce786 100644 --- a/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl +++ b/apps/emqx_bridge_opents/test/emqx_bridge_opents_SUITE.erl @@ -53,7 +53,7 @@ init_per_suite(Config) -> end_per_suite(_Config) -> emqx_mgmt_api_test_util:end_suite(), - ok = emqx_common_test_helpers:stop_apps([emqx_bridge, emqx_conf]), + ok = emqx_common_test_helpers:stop_apps([opentsdb, emqx_bridge, emqx_resource, emqx_conf]), ok. init_per_testcase(_Testcase, Config) -> @@ -91,10 +91,12 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, emqx_resource, emqx_bridge + ]), + _ = application:ensure_all_started(opentsdb), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), {Name, OpenTSConf} = opents_config(BridgeType, Config0), Config = diff --git a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src index ad96b4744..a05533da3 100644 --- a/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src +++ b/apps/emqx_bridge_oracle/src/emqx_bridge_oracle.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_oracle, [ {description, "EMQX Enterprise Oracle Database Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, emqx_oracle ]}, {env, []}, diff --git a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl index d7c7cec74..5c6eddb39 100644 --- a/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl +++ b/apps/emqx_bridge_oracle/test/emqx_bridge_oracle_SUITE.erl @@ -83,8 +83,9 @@ common_init_per_group() -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - application:load(emqx_bridge), - ok = emqx_common_test_helpers:start_apps([emqx_conf]), + %% Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src index 5a72107a4..ade791a6d 100644 --- a/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src +++ b/apps/emqx_bridge_pgsql/src/emqx_bridge_pgsql.app.src @@ -1,8 +1,13 @@ {application, emqx_bridge_pgsql, [ {description, "EMQX Enterprise PostgreSQL Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl index 6806328d6..d16488bc6 100644 --- a/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl +++ b/apps/emqx_bridge_pgsql/test/emqx_bridge_pgsql_SUITE.erl @@ -145,10 +145,9 @@ common_init(Config0) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), + % Ensure enterprise bridge module is loaded ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to pgsql directly and create the table connect_and_create_table(Config0), diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src index 487e862bc..99fb25c33 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src @@ -1,10 +1,12 @@ {application, emqx_bridge_pulsar, [ {description, "EMQX Pulsar Bridge"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {registered, []}, {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, pulsar ]}, {env, []}, diff --git a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl index 038da3e61..2fa5d70cf 100644 --- a/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl +++ b/apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl @@ -15,7 +15,7 @@ fields/1, desc/1 ]). -%% emqx_ee_bridge "unofficial" API +%% emqx_bridge_enterprise "unofficial" API -export([conn_bridge_examples/1]). -export([producer_strategy_key_validator/1]). diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index 4530748de..4e4914bc0 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -122,8 +122,10 @@ common_init_per_group() -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - application:load(emqx_bridge), - ok = emqx_common_test_helpers:start_apps([emqx_conf]), + %% Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_resource, emqx_bridge]), + _ = application:ensure_all_started(pulsar), + _ = emqx_bridge_enterprise:module_info(), ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), diff --git a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src index b8f7b3327..e9ef4d524 100644 --- a/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src +++ b/apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq.app.src @@ -1,8 +1,16 @@ {application, emqx_bridge_rabbitmq, [ {description, "EMQX Enterprise RabbitMQ Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, ecql, rabbit_common, amqp_client]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + ecql, + rabbit_common, + amqp_client + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl index e6a6c03fb..d3f31f5fa 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_SUITE.erl @@ -13,7 +13,7 @@ -include_lib("amqp_client/include/amqp_client.hrl"). %% See comment in -%% lib-ee/emqx_ee_connector/test/ee_connector_rabbitmq_SUITE.erl for how to +%% apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl for how to %% run this without bringing up the whole CI infrastucture rabbit_mq_host() -> @@ -50,8 +50,6 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), - {ok, _} = application:ensure_all_started(emqx_ee_bridge), {ok, _} = application:ensure_all_started(amqp_client), emqx_mgmt_api_test_util:init_suite(), ChannelConnection = setup_rabbit_mq_exchange_and_queue(), @@ -112,7 +110,6 @@ end_per_suite(Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_conf]), ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), _ = application:stop(emqx_connector), - _ = application:stop(emqx_ee_connector), _ = application:stop(emqx_bridge), %% Close the channel ok = amqp_channel:close(Channel), diff --git a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl index 6b6ad617f..106a4d67b 100644 --- a/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl +++ b/apps/emqx_bridge_rabbitmq/test/emqx_bridge_rabbitmq_connector_SUITE.erl @@ -48,7 +48,6 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), {ok, _} = application:ensure_all_started(emqx_connector), - {ok, _} = application:ensure_all_started(emqx_ee_connector), {ok, _} = application:ensure_all_started(amqp_client), ChannelConnection = setup_rabbit_mq_exchange_and_queue(), [{channel_connection, ChannelConnection} | Config]; diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src index 0375b6cd2..bc21adcad 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis.app.src @@ -1,8 +1,15 @@ {application, emqx_bridge_redis, [ {description, "EMQX Enterprise Redis Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, emqx_redis]}, + {applications, [ + kernel, + stdlib, + emqx_connector, + emqx_resource, + emqx_bridge, + emqx_redis + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl index 046c42180..38a80048e 100644 --- a/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl +++ b/apps/emqx_bridge_redis/src/emqx_bridge_redis_connector.erl @@ -28,7 +28,7 @@ on_start(InstId, #{command_template := CommandTemplate} = Config) -> case emqx_redis:on_start(InstId, Config) of {ok, RedisConnSt} -> ?tp( - redis_ee_connector_start_success, + redis_bridge_connector_start_success, #{} ), {ok, #{ @@ -37,7 +37,7 @@ on_start(InstId, #{command_template := CommandTemplate} = Config) -> }}; {error, _} = Error -> ?tp( - redis_ee_connector_start_error, + redis_bridge_connector_start_error, #{error => Error} ), Error @@ -60,12 +60,12 @@ on_query( ) -> Cmd = proc_command_template(CommandTemplate, Data), ?tp( - redis_ee_connector_cmd, + redis_bridge_connector_cmd, #{cmd => Cmd, batch => false, mode => sync} ), Result = query(InstId, {cmd, Cmd}, RedisConnSt), ?tp( - redis_ee_connector_send_done, + redis_bridge_connector_send_done, #{cmd => Cmd, batch => false, mode => sync, result => Result} ), Result; @@ -75,12 +75,12 @@ on_query( _State = #{conn_st := RedisConnSt} ) -> ?tp( - redis_ee_connector_query, + redis_bridge_connector_query, #{query => Query, batch => false, mode => sync} ), Result = query(InstId, Query, RedisConnSt), ?tp( - redis_ee_connector_send_done, + redis_bridge_connector_send_done, #{query => Query, batch => false, mode => sync, result => Result} ), Result. @@ -90,12 +90,12 @@ on_batch_query( ) -> Cmds = process_batch_data(BatchData, CommandTemplate), ?tp( - redis_ee_connector_send, + redis_bridge_connector_send, #{batch_data => BatchData, batch => true, mode => sync} ), Result = query(InstId, {cmds, Cmds}, RedisConnSt), ?tp( - redis_ee_connector_send_done, + redis_bridge_connector_send_done, #{ batch_data => BatchData, batch_size => length(BatchData), diff --git a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl index 242e74b3e..6a0248b67 100644 --- a/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl +++ b/apps/emqx_bridge_redis/test/emqx_bridge_redis_SUITE.erl @@ -117,11 +117,9 @@ wait_for_ci_redis(Checks, Config) -> ProxyHost = os:getenv("PROXY_HOST", ?PROXY_HOST), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", ?PROXY_PORT)), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - ok = emqx_common_test_helpers:start_apps([emqx_conf]), - ok = emqx_connector_test_helpers:start_apps([ - emqx_resource, emqx_bridge, emqx_rule_engine + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, emqx_resource, emqx_connector, emqx_bridge, emqx_rule_engine ]), - {ok, _} = application:ensure_all_started(emqx_connector), [ {proxy_host, ProxyHost}, {proxy_port, ProxyPort} @@ -271,21 +269,21 @@ t_check_replay(Config) -> lists:seq(1, ?BATCH_SIZE) ), #{ - ?snk_kind := redis_ee_connector_send_done, + ?snk_kind := redis_bridge_connector_send_done, batch := true, result := {error, _} }, 10_000 ) end), - #{?snk_kind := redis_ee_connector_send_done, batch := true, result := {ok, _}}, + #{?snk_kind := redis_bridge_connector_send_done, batch := true, result := {ok, _}}, 10_000 ), fun(Trace) -> ?assert( ?strict_causality( - #{?snk_kind := redis_ee_connector_send_done, result := {error, _}}, - #{?snk_kind := redis_ee_connector_send_done, result := {ok, _}}, + #{?snk_kind := redis_bridge_connector_send_done, result := {error, _}}, + #{?snk_kind := redis_bridge_connector_send_done, result := {ok, _}}, Trace ) ) @@ -308,14 +306,14 @@ t_permanent_error(_Config) -> begin ?wait_async_action( publish_message(Topic, Payload), - #{?snk_kind := redis_ee_connector_send_done}, + #{?snk_kind := redis_bridge_connector_send_done}, 10_000 ) end, fun(Trace) -> ?assertMatch( [#{result := {error, _}} | _], - ?of_kind(redis_ee_connector_send_done, Trace) + ?of_kind(redis_bridge_connector_send_done, Trace) ) end ), @@ -334,7 +332,7 @@ t_create_disconnected(Config) -> fun(Trace) -> ?assertMatch( [#{error := _} | _], - ?of_kind(redis_ee_connector_start_error, Trace) + ?of_kind(redis_bridge_connector_start_error, Trace) ), ok end @@ -365,7 +363,7 @@ check_resource_queries(ResourceId, BaseTopic, IsBatch) -> end, lists:seq(1, N) ), - #{?snk_kind := redis_ee_connector_send_done, batch := IsBatch}, + #{?snk_kind := redis_bridge_connector_send_done, batch := IsBatch}, 5000 ), fun(Trace) -> @@ -374,13 +372,13 @@ check_resource_queries(ResourceId, BaseTopic, IsBatch) -> true -> ?assertMatch( [#{result := {ok, _}, batch := true, batch_size := ?BATCH_SIZE} | _], - ?of_kind(redis_ee_connector_send_done, Trace) + ?of_kind(redis_bridge_connector_send_done, Trace) ), ?assertEqual(?BATCH_SIZE, AddedMsgCount); false -> ?assertMatch( [#{result := {ok, _}, batch := false} | _], - ?of_kind(redis_ee_connector_send_done, Trace) + ?of_kind(redis_bridge_connector_send_done, Trace) ), ?assertEqual(1, AddedMsgCount) end diff --git a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src index 7da5430a9..e18b98e3a 100644 --- a/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src +++ b/apps/emqx_bridge_rocketmq/src/emqx_bridge_rocketmq.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_rocketmq, [ {description, "EMQX Enterprise RocketMQ Bridge"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, - {applications, [kernel, stdlib, rocketmq]}, + {applications, [kernel, stdlib, emqx_resource, emqx_bridge, rocketmq]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl b/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl index 62e1a7b3f..1a5133b84 100644 --- a/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl +++ b/apps/emqx_bridge_rocketmq/test/emqx_bridge_rocketmq_SUITE.erl @@ -109,10 +109,11 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([ + emqx_conf, emqx_resource, emqx_bridge, rocketmq + ]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), {Name, RocketMQConf} = rocketmq_config(BridgeType, Config0), Config = diff --git a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src index e5c5ae73d..35f4587b0 100644 --- a/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src +++ b/apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_sqlserver, [ {description, "EMQX Enterprise SQL Server Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib, odbc]}, + {applications, [kernel, stdlib, emqx_resource, emqx_bridge, odbc]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl b/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl index 0e60e9c97..101ead838 100644 --- a/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl +++ b/apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl @@ -416,10 +416,9 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge, odbc]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to sqlserver directly % drop old db and table, and then create new ones diff --git a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src index 97d8ff2e5..e4c946162 100644 --- a/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src +++ b/apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine.app.src @@ -1,8 +1,14 @@ {application, emqx_bridge_tdengine, [ {description, "EMQX Enterprise TDEngine Bridge"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, - {applications, [kernel, stdlib, tdengine]}, + {applications, [ + kernel, + stdlib, + emqx_resource, + emqx_bridge, + tdengine + ]}, {env, []}, {modules, []}, {links, []} diff --git a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl index 9399f6029..54744d806 100644 --- a/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl +++ b/apps/emqx_bridge_tdengine/test/emqx_bridge_tdengine_SUITE.erl @@ -142,10 +142,9 @@ common_init(ConfigT) -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), - % Ensure EE bridge module is loaded - _ = application:load(emqx_ee_bridge), - _ = emqx_ee_bridge:module_info(), - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge]), + % Ensure enterprise bridge module is loaded + ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_bridge, tdengine]), + _ = emqx_bridge_enterprise:module_info(), emqx_mgmt_api_test_util:init_suite(), % Connect to tdengine directly and create the table connect_and_create_table(Config0), diff --git a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src index f533f3b04..7a4aeeb56 100644 --- a/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src +++ b/apps/emqx_bridge_timescale/src/emqx_bridge_timescale.app.src @@ -1,8 +1,8 @@ {application, emqx_bridge_timescale, [ {description, "EMQX Enterprise TimescaleDB Bridge"}, - {vsn, "0.1.1"}, + {vsn, "0.1.2"}, {registered, []}, - {applications, [kernel, stdlib]}, + {applications, [kernel, stdlib, emqx_resource, emqx_bridge]}, {env, []}, {modules, []}, {links, []} diff --git a/lib-ee/emqx_ee_bridge/.gitignore b/lib-ee/emqx_ee_bridge/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/lib-ee/emqx_ee_bridge/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/lib-ee/emqx_ee_bridge/README.md b/lib-ee/emqx_ee_bridge/README.md deleted file mode 100644 index 5cb4d8694..000000000 --- a/lib-ee/emqx_ee_bridge/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_ee_bridge -===== - -An OTP application - -Build ------ - - $ rebar3 compile diff --git a/lib-ee/emqx_ee_bridge/docker-ct b/lib-ee/emqx_ee_bridge/docker-ct deleted file mode 100644 index 80f0d394b..000000000 --- a/lib-ee/emqx_ee_bridge/docker-ct +++ /dev/null @@ -1 +0,0 @@ -toxiproxy diff --git a/lib-ee/emqx_ee_bridge/rebar.config b/lib-ee/emqx_ee_bridge/rebar.config deleted file mode 100644 index 3b3be6ccf..000000000 --- a/lib-ee/emqx_ee_bridge/rebar.config +++ /dev/null @@ -1,11 +0,0 @@ -%% -*- mode: erlang; -*- -{erl_opts, [debug_info]}. -{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}} - , {emqx_resource, {path, "../../apps/emqx_resource"}} - , {emqx_bridge, {path, "../../apps/emqx_bridge"}} - , {emqx_utils, {path, "../emqx_utils"}} - ]}. - -{shell, [ - {apps, [emqx_ee_bridge]} -]}. diff --git a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src b/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src deleted file mode 100644 index e03cc9423..000000000 --- a/lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src +++ /dev/null @@ -1,27 +0,0 @@ -{application, emqx_ee_bridge, [ - {description, "EMQX Enterprise data bridges"}, - {vsn, "0.1.16"}, - {registered, []}, - {applications, [ - kernel, - stdlib, - emqx_ee_connector, - telemetry, - emqx_bridge_kafka, - emqx_bridge_gcp_pubsub, - emqx_bridge_cassandra, - emqx_bridge_opents, - emqx_bridge_pulsar, - emqx_bridge_dynamo, - emqx_bridge_sqlserver, - emqx_bridge_rocketmq, - emqx_bridge_rabbitmq, - emqx_bridge_tdengine, - emqx_bridge_influxdb, - emqx_bridge_clickhouse - ]}, - {env, []}, - {modules, []}, - - {links, []} -]}. diff --git a/lib-ee/emqx_ee_connector/.gitignore b/lib-ee/emqx_ee_connector/.gitignore deleted file mode 100644 index f1c455451..000000000 --- a/lib-ee/emqx_ee_connector/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -.rebar3 -_* -.eunit -*.o -*.beam -*.plt -*.swp -*.swo -.erlang.cookie -ebin -log -erl_crash.dump -.rebar -logs -_build -.idea -*.iml -rebar3.crashdump -*~ diff --git a/lib-ee/emqx_ee_connector/README.md b/lib-ee/emqx_ee_connector/README.md deleted file mode 100644 index e665af458..000000000 --- a/lib-ee/emqx_ee_connector/README.md +++ /dev/null @@ -1,9 +0,0 @@ -emqx_ee_connector -===== - -An OTP application - -Build ------ - - $ rebar3 compile diff --git a/lib-ee/emqx_ee_connector/docker-ct b/lib-ee/emqx_ee_connector/docker-ct deleted file mode 100644 index ef579c036..000000000 --- a/lib-ee/emqx_ee_connector/docker-ct +++ /dev/null @@ -1,2 +0,0 @@ -toxiproxy -influxdb diff --git a/lib-ee/emqx_ee_connector/rebar.config b/lib-ee/emqx_ee_connector/rebar.config deleted file mode 100644 index 1f52a4f03..000000000 --- a/lib-ee/emqx_ee_connector/rebar.config +++ /dev/null @@ -1,10 +0,0 @@ -%% -*- mode: erlang -*- -{erl_opts, [debug_info]}. -{deps, [ - {emqx, {path, "../../apps/emqx"}}, - {emqx_utils, {path, "../../apps/emqx_utils"}} -]}. - -{shell, [ - {apps, [emqx_ee_connector]} -]}. diff --git a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src b/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src deleted file mode 100644 index c3187f807..000000000 --- a/lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src +++ /dev/null @@ -1,16 +0,0 @@ -{application, emqx_ee_connector, [ - {description, "EMQX Enterprise connectors"}, - {vsn, "0.1.15"}, - {registered, []}, - {applications, [ - kernel, - stdlib, - ecpool, - hstreamdb_erl, - emqx_redis - ]}, - {env, []}, - {modules, []}, - - {links, []} -]}. diff --git a/mix.exs b/mix.exs index 99aacdf1d..9e9279895 100644 --- a/mix.exs +++ b/mix.exs @@ -395,8 +395,6 @@ defmodule EMQXUmbrella.MixProject do do: [ emqx_license: :permanent, emqx_enterprise: :load, - emqx_ee_connector: :permanent, - emqx_ee_bridge: :permanent, emqx_bridge_kafka: :permanent, emqx_bridge_pulsar: :permanent, emqx_bridge_gcp_pubsub: :permanent, diff --git a/rebar.config.erl b/rebar.config.erl index 312fc1173..5f86afaa2 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -463,8 +463,6 @@ relx_apps_per_edition(ee) -> [ emqx_license, {emqx_enterprise, load}, - emqx_ee_connector, - emqx_ee_bridge, emqx_bridge_kafka, emqx_bridge_pulsar, emqx_bridge_gcp_pubsub, From 4ee44972b2a274370baa1ec37377c34c993cef7b Mon Sep 17 00:00:00 2001 From: JimMoen Date: Thu, 6 Jul 2023 11:42:22 +0800 Subject: [PATCH 41/92] feat: refactor hstreamdb connector to to avoid resources leaking --- .../src/emqx_bridge_hstreamdb_connector.erl | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl index 31627db81..16092f262 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb_connector.erl @@ -33,6 +33,9 @@ desc/1 ]). +%% Allocatable resources +-define(hstreamdb_client, hstreamdb_client). + -define(DEFAULT_GRPC_TIMEOUT, timer:seconds(30)). -define(DEFAULT_GRPC_TIMEOUT_RAW, <<"30s">>). @@ -43,17 +46,22 @@ callback_mode() -> always_sync. on_start(InstId, Config) -> start_client(InstId, Config). -on_stop(InstId, #{client := Client, producer := Producer}) -> - StopClientRes = hstreamdb:stop_client(Client), - StopProducerRes = hstreamdb:stop_producer(Producer), - ?SLOG(info, #{ - msg => "stop hstreamdb connector", - connector => InstId, - client => Client, - producer => Producer, - stop_client => StopClientRes, - stop_producer => StopProducerRes - }). +on_stop(InstId, _State) -> + case emqx_resource:get_allocated_resources(InstId) of + #{client := Client, producer := Producer} -> + StopClientRes = hstreamdb:stop_client(Client), + StopProducerRes = hstreamdb:stop_producer(Producer), + ?SLOG(info, #{ + msg => "stop hstreamdb connector", + connector => InstId, + client => Client, + producer => Producer, + stop_client => StopClientRes, + stop_producer => StopProducerRes + }); + _ -> + ok + end. -define(FAILED_TO_APPLY_HRECORD_TEMPLATE, {error, {unrecoverable_error, failed_to_apply_hrecord_template}} @@ -237,6 +245,9 @@ start_producer( ), record_template => record_template(Options) }, + ok = emqx_resource:allocate_resource(InstId, ?hstreamdb_client, #{ + client => Client, producer => Producer + }), {ok, State}; {error, {already_started, Pid}} -> ?SLOG(info, #{ From 30c931ae62ee57c78d776a8a84fc886d888c517e Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 7 Jul 2023 09:52:41 +0800 Subject: [PATCH 42/92] fix: pulsar flaky cluster tests --- .../test/emqx_bridge_pulsar_impl_producer_SUITE.erl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl index 4e4914bc0..15d4b63d4 100644 --- a/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl +++ b/apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl @@ -14,7 +14,7 @@ -import(emqx_common_test_helpers, [on_exit/1]). -define(BRIDGE_TYPE_BIN, <<"pulsar_producer">>). --define(APPS, [emqx_bridge, emqx_resource, emqx_rule_engine, emqx_bridge_pulsar]). +-define(APPS, [emqx_resource, emqx_bridge, emqx_rule_engine, emqx_bridge_pulsar]). -define(RULE_TOPIC, "mqtt/rule"). -define(RULE_TOPIC_BIN, <>). @@ -123,10 +123,10 @@ common_init_per_group() -> ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), %% Ensure enterprise bridge module is loaded - ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_resource, emqx_bridge]), - _ = application:ensure_all_started(pulsar), + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_common_test_helpers:start_apps(?APPS), + {ok, _} = application:ensure_all_started(pulsar), _ = emqx_bridge_enterprise:module_info(), - ok = emqx_connector_test_helpers:start_apps(?APPS), {ok, _} = application:ensure_all_started(emqx_connector), emqx_mgmt_api_test_util:init_suite(), UniqueNum = integer_to_binary(erlang:unique_integer()), @@ -520,7 +520,7 @@ cluster(Config) -> Cluster = emqx_common_test_helpers:emqx_cluster( [core, core], [ - {apps, [emqx_conf, emqx_bridge, emqx_rule_engine, emqx_bridge_pulsar]}, + {apps, [emqx_conf] ++ ?APPS ++ [pulsar]}, {listener_ports, []}, {peer_mod, PeerModule}, {priv_data_dir, PrivDataDir}, @@ -1099,6 +1099,7 @@ do_t_cluster(Config) -> ), {ok, _} = erpc:call(N1, fun() -> create_bridge(Config) end), {ok, _} = snabbkaffe:receive_events(SRef1), + erpc:multicall(Nodes, fun wait_until_producer_connected/0), {ok, _} = snabbkaffe:block_until( ?match_n_events( NumNodes, @@ -1120,7 +1121,6 @@ do_t_cluster(Config) -> end, Nodes ), - erpc:multicall(Nodes, fun wait_until_producer_connected/0), Message0 = emqx_message:make(ClientId, QoS, MQTTTopic, Payload), ?tp(publishing_message, #{}), erpc:call(N2, emqx, publish, [Message0]), From c4d8e04efeae4a48e3038e30ae1ca612f7d05e0e Mon Sep 17 00:00:00 2001 From: JianBo He Date: Wed, 21 Jun 2023 10:47:22 +0800 Subject: [PATCH 43/92] chore: update license expiry log --- lib-ee/emqx_license/include/emqx_license.hrl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib-ee/emqx_license/include/emqx_license.hrl b/lib-ee/emqx_license/include/emqx_license.hrl index b2a0bb40b..2bd5ee9ec 100644 --- a/lib-ee/emqx_license/include/emqx_license.hrl +++ b/lib-ee/emqx_license/include/emqx_license.hrl @@ -12,7 +12,7 @@ "========================================================================\n" "Using an evaluation license limited to ~p concurrent connections.\n" "Apply for a license at https://emqx.com/apply-licenses/emqx.\n" - "Or contact EMQ customer services.\n" + "Or contact EMQ customer services via email at contact@emqx.io\n" "========================================================================\n" ). @@ -21,7 +21,7 @@ "========================================================================\n" "License has been expired for ~p days.\n" "Apply for a new license at https://emqx.com/apply-licenses/emqx.\n" - "Or contact EMQ customer services.\n" + "Or contact EMQ customer services via email at contact@emqx.io\n" "========================================================================\n" ). From 8e4bef9cb10f59b7e2a13a89d03a221014222783 Mon Sep 17 00:00:00 2001 From: JianBo He Date: Thu, 6 Jul 2023 16:36:14 +0800 Subject: [PATCH 44/92] chore(license): bump version --- lib-ee/emqx_license/src/emqx_license.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib-ee/emqx_license/src/emqx_license.app.src b/lib-ee/emqx_license/src/emqx_license.app.src index 2f21b8a52..f3a614a70 100644 --- a/lib-ee/emqx_license/src/emqx_license.app.src +++ b/lib-ee/emqx_license/src/emqx_license.app.src @@ -1,6 +1,6 @@ {application, emqx_license, [ {description, "EMQX License"}, - {vsn, "5.0.11"}, + {vsn, "5.0.12"}, {modules, []}, {registered, [emqx_license_sup]}, {applications, [kernel, stdlib, emqx_ctl]}, From 63c64270372152ee7742244551160ea8159d5a97 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 7 Jul 2023 14:50:44 +0800 Subject: [PATCH 45/92] chore: hstreamdb_erl as deps for hstreamdb bridge --- apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src index 2b1e96b00..0549dd020 100644 --- a/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src +++ b/apps/emqx_bridge_hstreamdb/src/emqx_bridge_hstreamdb.app.src @@ -6,7 +6,8 @@ kernel, stdlib, emqx_resource, - emqx_bridge + emqx_bridge, + hstreamdb_erl ]}, {env, []}, {modules, []}, From 352d818e3b14a64259287cda7e9a763d26569f96 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 4 Jul 2023 20:17:07 +0800 Subject: [PATCH 46/92] chore: update hocon to 0.39.13 --- apps/emqx/rebar.config | 2 +- changes/ce/fix-11077.en.md | 1 + mix.exs | 2 +- rebar.config | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-11077.en.md diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index 0278a1b1d..78a8b76e0 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -29,7 +29,7 @@ {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}, {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.5"}}}, {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}, - {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.11"}}}, + {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.13"}}}, {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}}, {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}, {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}, diff --git a/changes/ce/fix-11077.en.md b/changes/ce/fix-11077.en.md new file mode 100644 index 000000000..64911df4a --- /dev/null +++ b/changes/ce/fix-11077.en.md @@ -0,0 +1 @@ +Fixes crash when updating binding with a non-integer port. diff --git a/mix.exs b/mix.exs index 9e9279895..65c942394 100644 --- a/mix.exs +++ b/mix.exs @@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do # in conflict by emqtt and hocon {:getopt, "1.0.2", override: true}, {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true}, - {:hocon, github: "emqx/hocon", tag: "0.39.11", override: true}, + {:hocon, github: "emqx/hocon", tag: "0.39.13", override: true}, {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true}, {:esasl, github: "emqx/esasl", tag: "0.2.0"}, {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"}, diff --git a/rebar.config b/rebar.config index ff55d2e70..0f6864c5e 100644 --- a/rebar.config +++ b/rebar.config @@ -75,7 +75,7 @@ , {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}} , {getopt, "1.0.2"} , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}} - , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.11"}}} + , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.13"}}} , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.2"}}} , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}} , {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}} From 7039dca51c7eac9ec5fbffa9ef5d5c0a1edf59bc Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Tue, 4 Jul 2023 20:25:59 +0800 Subject: [PATCH 47/92] chore: add 11192 changelog --- changes/ce/fix-11192.en.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 changes/ce/fix-11192.en.md diff --git a/changes/ce/fix-11192.en.md b/changes/ce/fix-11192.en.md new file mode 100644 index 000000000..6bbc0b12d --- /dev/null +++ b/changes/ce/fix-11192.en.md @@ -0,0 +1,2 @@ +Fix produces valid HOCON file when atom type is used. +Remove unnecessary `"` from HOCON file. From b5cc8fb3c3abddba651fbc218750b056ee526cbc Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 7 Jul 2023 16:38:40 +0800 Subject: [PATCH 48/92] fix: start_after_created's default value --- apps/emqx_authz/test/emqx_authz_test_lib.erl | 2 +- .../emqx-export-test-bootstrap-ce.tar.gz | Bin 11822 -> 11973 bytes apps/emqx_resource/include/emqx_resource.hrl | 1 - apps/emqx_resource/src/emqx_resource.app.src | 2 +- .../src/schema/emqx_resource_schema.erl | 2 +- 5 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/emqx_authz/test/emqx_authz_test_lib.erl b/apps/emqx_authz/test/emqx_authz_test_lib.erl index e308e0de7..03d687851 100644 --- a/apps/emqx_authz/test/emqx_authz_test_lib.erl +++ b/apps/emqx_authz/test/emqx_authz_test_lib.erl @@ -35,7 +35,7 @@ reset_authorizers(Nomatch, CacheEnabled, Source) -> [authorization], #{ <<"no_match">> => atom_to_binary(Nomatch), - <<"cache">> => #{<<"enable">> => atom_to_binary(CacheEnabled)}, + <<"cache">> => #{<<"enable">> => CacheEnabled}, <<"sources">> => Source } ), diff --git a/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-test-bootstrap-ce.tar.gz b/apps/emqx_management/test/emqx_mgmt_data_backup_SUITE_data/emqx-export-test-bootstrap-ce.tar.gz index b7da76bbb0a8b1037f64e21304a4376f21ff08ab..a22ff40a5708e515ac1d61fd61102980dc443aa6 100644 GIT binary patch literal 11973 zcmaLdMNk|7wQ6xrQ@H|0!8)AZNGo7&am28dq2X8}lkg!ulC;?-TpA358vdB;7+TeY1X$3T4z{p(N7rSe`4*p}Bj67B5p!wr|f%MAb^ z+>{U3VGB$6Vy^=qWymr)7ABzbhFOwwun&MWfQ5)zj^z|vLXNMff_OnLDp+trYNxlp z%b1BBhevXX?wun~vfDcjy1IrR4wPE%D;(VWvh25tf$~wgA9yH2g$Y~~64}F6AYKs) z0$J^m2Md6Fw=8=n&I>$<2~Oh4Z{kf(l4d0Z186Y4pdbn2qunfDI}t9KKy|-)DI`dv zx`z_H0D5qO9N)*f>%iI9paZ*gb_S<|u9%{Tyu7LPyGSjnAyp!s82&jg)EDMeHl@VH z!_CO)Aq4p$O|J1jEt})wItGF$gHCovaoA1=HB&pBkX7`-gu~rU{pja!!a{5{e|_Ms zvb0B4cxTOw&v=SzQDKm|d($9(p-M%Q<}Vm#$+EG5O!(xnU%c~fs!-c1i4}UX{LDI_ z(%E$4VA8ZW`0jRgDO7uDQ_Cx1bQ?S`OVrVV=QN-6Pz~4NFtulie;s6;`sme`8COu~ z>bzf5e$DAXu1UMDA4%(QedjJ?`6Jz?Q!VZ;6Yt5^AZ)fyzSHF(1Qz6vO+`)<X4vkcfJG^^a}UDF9#vUFNCt(*-;0cM?Xl4L_;qlHpu!YOzrIo0a;6rdeq zCPuF;3wQ@jLK)@-gnt%w4=C)maZ%XI{oKsV$V(Hcw^er3D&x`yZfiP^zFlrcQWqCx z@-_1fJsc>9vJtxn-1V7ON*xwqGT}}G7dIU>e6cyB+*>*pZ1UqutdsaGYMa*O(@P&4 zum1I}qxa6ljI9_QbjXvTjPhA@^wURo^5doavF3z&>BiPpWl3hf&n@b~eLNP7(wl7B zds@6e`K2oL;By*06(viBmDZw4N6wtej#~e?>d_u2Hr+FZh2MG27kh!FUQN1DIH(cd z>BFVKk#QnvxGqv-!7+J3CF;a#`8atfpF)dt~+L z&TP518jb7}^PI8_57nF%*#VkC*pXrB4j8zWXpfri{E?XZX4OKmqf5KWKg)kJY~w!+ zOF9gCcJR=K9xqgJ=3WF2u_(h;!t5}K7cm2Lpqvocd@)h-cNJ?@!~5nCle zn8j?=^+;uUa)X_Ywv#jHx5FF*4{(PB`E~56JfQoqnP4@^gXGQG2&j|s@V^N&Id*pjkLf1>V8W7_CYuW#MINYc6F{gsu5ccI!(JZwvwQf_1|BYBp!d=V>G<1z4IF~;UuRgj#ZNG;e*nL=d|oPi zxX9Zx0QuCKjlQswnPu4~ktD!prRq7GR=d@2UU)Z-)>3~*rV$y;|MmrU)!B~T%B zwqWF5P@Yb-+SC2jbIe-u{Yzb3rcR6!H6XsHW>N+oej~yYuYVf0)nvU=t0=J6 z9zm_EH_{@Za5N^lT?psooR>9QE>kYd&J%aq0mGUm_`XUnfT5(v=^|;SZ(q8RGRAF> zYQ8bNIl-N7RZzJ-Wl%;Wi;X|1HkK+D1ubTT^&#K2qrpa8pMGn;4pqMg*HR7RzH~Y> zQ(fQT9F_%J)Zk3D*^eowonco-GD@W}&!a>RGz$WkoCPL_4J3e96>41yj57 ztXtX8Ts8gNd6mXwVMNe^)O)~~9&0ROfS>`+*J&Ftql~I%@AVmy9@@Sc(GcabaF@Yv z%DN4dHDPiqZP$faC3D&|zh$65ptI&086}D>>G4+3w=m@t0WiaxjeNDB55i;b=sc(* z+A5_f+bp2%p(kecz_y!<<9W)>t7(Dm=Wr7Sv{Ex?lp*ynaYmx)o>-Wx=yRmk)#0*r zN|~#b8M~KgYz1{xjK$NT%E_2+J={$a(#F$-^8q5&SuG@j7RZlb%;h#$Df7$3;ca3# z^)_ColCs43u1D=gRYKRJws|u>LDQ+sBga)c51Edk zN~~td+Dh%#offv3Fe0rw*TN|y(>?s~`Wu`dMZsKn(uKkB>&85(E;4=O^9)9c=QSwr zn+?m_;W!5aO=dRY#VBIQ)7Eti(1t~{-QA?30b zF9Cod)1;BI^~>rv5g!FN%(LGU9h2H7XJ}YuT(dMswYH7=y1<{VIw?PYwtkPsSI{;n zz^Q+j`DuB$;;ynsw{JEdm&Sg0iVXyC?4w}WMJVA$T!wvb)Ulf|iHjFiDADm3Cg9nM zdcMR1p`Rv^ zw6tF-O+QORgAe!BsKWc%$;p+YZq>C^GfWfPu6V^;cM|W&-WhwA}f-sl(W6UNxJ|-_zD!HEoY1(g%EbH9&2* zH*G;ehjV0Vna85Q6PocQ$E3WOWrou9@;n`L3aaoiW9fum6j@p zu!Z}}zhU)oJk68!*UvGD1RIu(>?O__E>ibkX&4g^{@AUYAmtgXARVelYaH*9=agsI z)r3ePa-FA%kf-N*m2G6#M?3z=ppMI)H){57)4jA=m8=Hj{ngx*H9<0{kSO+2E|HCM z)E>ZZ7>%}UdKj$HW;gMzF>pxMniL3o(uae8U#IK?GQ%xgrg+M{cTVl#= z>S2-5QM(6_qUuZEp@?55GqVeY$Ys-}7j-McNUW=fplJVWgJUC1Ps^?uc|3l>!V1F% zWcH-((xNMcv-wfUA3Skynr`I6NsJ_`3S55F48fHDou!Ls)yAUR7T(}calYBs61{@o zZlD%T|4Su$qNB-H>($*gkx+{rS?#iI*U|X|b#2p3aT#&cvzcMB$u?=u(!urqE3b5t z(I8L0YEu>6h24p$T+N8ABB((XKC-J^nGP7rrO`h%$d!@g{zRve`)Y?Pi;8D$f z$F(!`*cwZ0N4n*5&9vyrmXXqqM4O>p@c3f}TS~%Uyo*%}-@o6$#SSlK!7Mjk4ndm~ zB3820A8lNoG?T4wUPjd;OT037Q~fpNk^HzNSx}TNrRcG(V$k^mF7!I`ZGop#e-ur0 zv}&y3A8UTsoW)gw za5QwrwIZv2jutt(iq~YpsEUT2swb(n5sb&VdLe(p)3n8O zEWnh9w=%zB2dH*8w{E{YBjl7jD3Wv!Q2| z*Y-yh3n2`Xr}d~hJTHHOu|@$s$D4;t1<686U*R+vXTEEmCRn%xA;r{@MMpftBG-AF#PN||4Fb|fCW$}P<+B^# zz<-wXuxah)CeAhw=>&&P#hJFQ%4pcvojqym@%6b@3NwhK)QSDsS~&pj5ScVefaAf+ znCnVj$%1yn=IeKYA&t$U+ba&%ET;x`!lM|oU({tzP!DS*ODRj5Uh}lnyD$L57jk4kO|!Vks;AJFsE^?)d@iRKe2vN8KpZ5R-I`Vf^9 zi-}K~ar9LOrc=YC7ih(;FiJ&I;Zz9!cVy#MFO!q`;;=}4$X+k3meT*HuE8>~`PxHv`> zvgU4a>&7I(!r;Fpsp13+BD{S+2`NPX-!XfyHSL75pp0sTw6Cq z;#6-p0eV*Wr8Mq~^;W1G+CyR#cTkHb#+-;S{1;*v*Z&Q9&jIym7u-ij$|PcX5+t^b z(iP>+e%_v5z&lISd@M)o!o*7%4r2%(8xB@QblnN2>qw)cS~`m7H*NQW0V{-ikKtWKFry@HeO zSsTN4)W{P;WH|MYH@@qw+u0H}26JK(nZuQkeq=WA;@Bfu^eA&XVmuaEsY|f-&CL(W zmp+7trO5456WU~Z?I6um`&f>xZwH0g8cR36a_f$aU%)vF4di^}Zm27_!JT%ND>0Y6 z(#TkjKoe*=HCw)yOZ+l&q8$$W0}87(u3o7E9as5z3^_3d)JBfsdBFxk1g+>w;Kb$9ZERH@o)3?hFzcy_f9hLN43S-qT%v2iyLO z$m+sy$$q{A;6+bw-tEYe;{Z~ZBf#duXySsLf!fsFeHM5P;E~%0|icfRM1LX zgAE=d4`E}y2)&Mp9e#zMVi?0wUN^PfO#bJaW5=^$t+Nj@+Sh{Jv>ZdhK?3D-o*F&r z&xy-ntL>YOjM@|U{d4jjHFRw#0eL?3J6^?(%*SO}w&y$D+i~Ty1-%I8$WuUHM~9D_ zO`D<%D-(RxK-S%J!Tau`RY)RN?cq&NS5jfwwWOyARNEzFjhe}l2jp*zl2dl)D>zF3 zA(D8~^*&vnlkVmAS}wko4uW%taxl~v^!4;G#A%??*YDjVY%$q(c6+@VdY9U@?K`Lz zAPK97ml5)weRjI(G_(LH9q)GlmSu0|hf<_JOgj4TQgsne&>v@!Iswn{+bv=*PR|Gh z_Rw1qKJlu$6w??(nVyiggjz#nm>47q0C%yS-X{(j2xd*N`PcK+Qx^Bekk3BTR=KvM zr?13Daqo$vhdbroCEL{*mk>{7$)(&%y*_v4XojDLQN8}VD{K#KgxcmI=wiAZgn9Ib zOO5H;`F#ANzctQhg`t%%9B0a zoXxq9)08B#j3pT&BV|z-WlN%Kb*k9^+(U7b=R`4oN{jw?C~z*S`f?mPc#+mXS|pcS zlH!AN?QzZjz!@F)H-Gn?n__rx3eKJeN2TGBt8E=dxsA_Chjcvt_!J^tN)rr|fO{OZo>EBbx&anvhI)^PJe_I_+-arHQdx?V2#sR$x z{hPQ3$^Cgcxw)QrBvILZvSk5ysZvg%>n9%F69$8-{KJoL79uTfk;YVy#L=p z1mi?xK}*F&LB=nhYQ$cYW5Sulue`z@Nn>0+6iR<3hWJE1jN?bI%%HIZ!tXCIGLOL@ z=es>E>nL|mvUp7BRkwt>e9?SUP(rtp;tf#OP`9qB_})ClJ=cKQ28){|F{VuJN+B?u z(VP9)NB*C)Q7{_}KG+X3f8cRr#ukrHZW0+uMjjsd|FHKyjsZK8Di#(u+t<~M9dtT2 z5c!C^{9`C9rF7`loEywfEJcnTrA#TBgdp)>IDp5Hs0!y$9e}PN8+XM1Ld1r@0!IOW zneckz?1qL}fu=w{5e{Y^B)MAzQSkiQ=|+9I&xp5XzInf$>`Ize+PmANAh~x0p-V-R zDEF%fBNjwa=bzpJgAAo&nPGky9H-}KU6b@Lf!p=+n?`bY$oIm2dF{sR`90(8^WOPj zDopMy>b>;Ak2V=k%{V(7{=E}Nem&HvM!9fbE9S4ULgj4Irx26ICo>KW{e~&|KTw%Q zo*tvkWYQ6N_+Pk+5iB?m_FD9(!9@m_*@9>IoGjz*eeXm-h{h5`XG0@J`LNhC}WXK1`X~9CG>Ab{qN~tyi?|`QZq6`9tq9X2iJB|94yq40)|9?A&@J zYh&_lMKRu@-Mnac7Nrj401S4wxeg1^=r8JX?xqJu4B=@Ym$U3O2Y;SEp&y3!Q4+4@ z(Cg#^h7lu#r{8FjzuHZ@2bOp8(T3` zVQ}ywHOcpITIc5MPs))=d954QC#F>! z?R)o643i4g+5B$|r6$u{wi)NX@@Y1Olo^j(glL6yEe|RNt#Pym+H7893xftmd+N_hgjcbajW6fxh`R1uCktRgi|W5nc2{; z(vkp%ebV$&amVVmE|K?X*Y!XzJSuM1-o>mM7Nx3R+lm>|hkeg(H~z?clYf|)JIKlM zE!Mci{^P87>+HuwrR-K~1neZ*U5m$2xMyFEZI3|2`RcveF`uSqKIqo0mLZGS8rsNa z(IcU*DU(Tvt-9{<&_$wtA*6r+~{wp-q&Vv5NYuF0+Q2&c^Yi+!ZcO$3>6{F)sRq zF-}5#x;bQ?zm?=Ud$fC_hLtzVlJi*AR1%R}2fHImy;?!{4qP{l+A z{AF#@!(}s8oe`xgS(6gUqzF{y#-so+ueNH;(k9j7uDecsVKq;N6Y1B~H<76&)F9-? ztNkL%;!xhlVHNt-%a^F z*TANoG#`^?U!QpSknPB!ZIC7ZRgu?P&W%`Yy|o66i5LDA7fO?q2&RP&rl5L(Nk2xS z%Uy8AE|yQ+<0m6ErWbu$Z9RR8Wda43%M4D0s_bys>-?E3?!Bshe!q?zr%I8}bS0Jo z+rRHX84clHsbz~+XLCmSq2W}6vEey7vj+z0CZ62oJLd#V5$!nW?F6RPQr3 zQT;I^9KB8s5k+hBT+7mO@#9)!8!7sM3y7)`u62I>4791T2FS@%+bTwC)jt?8S)lx6 z)wARgw!&fHYmP3xxgcMWr+01ZodWd)r?y{;Ox;$b#+B0``zu!As@VNnnH0T}H7vuk zXdI5JmGPLD#W}eX*0I{Gyc|YuwU{cd1heo-BPs~;xVA`EuQYSkY_jH9`O+cnXStW6 z$NfG<@wOJAYwb3-HPM8_0%hYgcl=bD!MjIY)omO1k1b#e!_8DG->O;Ji#Au91&R|X z4Pj`Cr%G2gh%d5nm3 zvLw%YzyhT8R)q#*39#O52OhfV&@@(~-iu3QY$~&`?;%eDb0lQ+Tcv6h(cwheH>_ zq(Win*SW6_9$AusO8QyJd`6A)~F=h=k3g6I!{i z)~U5l@Yn-47%JASTi4H2*1Au)$HD-n(Kde zC(X)P+tOqSzGqk}t(yOCr5>*FEY&@aXJg_;)L*<0ljzK5;Ide-MpODh+fi}$*H_PG z&E1)eqzo6;DL)1?=4Zk<%Fpvwea1vb)xT3{CwuB{a-tp`7Glk+)ag}xZy#*u-K5R<^)u(qc`)MbQMHeFk!@*9bX66O}&7i{@!g| zZOrlZ`mg5)Th7df5Xu;%!LBpb(=}+uMIn~AA6hSQF2AP%E0FulMl&Eo*l zwl&psoPx{R#$N#>^rWtbr5@3UO{(wZivC1K1Vl*B;ArEO`r#gTyoP!lcRPoz$+|N) zZ?RhyTjNA{4=nhMWuY_YJ?Rc*+7^nJ;+By*ebv~O5sX*y1FWRkdcwP?mXX9Z{CQll zH4|xF6ss6AJJu4$@c#~d44*KsXh%!U@!|10w`!=U?>j4?Z33iIe!#WU=U_INLya0olr%Oric+a^d`y1-+yz%(Xr73U_`}v{P5wv9Yp&Z2@R)tZ=IsF-XXvxbbT?{eiKTnT=hDkcTV!rtbDRrW;TC%hZY|g z%#JYgojYU)C74~B43@U!dl+^};vL(MlJ84cCEmg6N{MHPssbWaG^B7#Dfd4{`?xML zJ1*LXa+}hdb4S`+E|XySf*rd6^;O|`L69r=j=X%aRQaoR2Rm%Nf742Dx1!P~=@LTtc<1|gs?faFk%9$?4B2%Fr^nXWb6(?J9KfMkv8R0S)%vGQy$Hx zMFsEFb2GE9_MYc&e9^QH_P^F0NU+smPiDq~(`X3d2OXaDh-<0ycw~&!j)F_WGfwr_ z#b32SDAI$G1>AE>D-uoS{_ftWdKIPEe>VQWt7(q*-lcpUY=~#}3YiJF?>`CYPC$`^ zS$QHPiJhOpqk3@EORge|Q*VG8P*e&KD}Z{6U*XCZJStlNZ8&gGg7s&(F}0SrusmA`g5-B+Tz-i{J-yAmN*85g9kb9LEhYwwPP z1;Oz56x{#Cyj9=$HfNhsNSD3Weh=T}!%5mx`Ye#77fDM|7MsatGVvdT36HHlK&en@g3S9XVG1f6D)`^CxZ>YV>XebM~~ zR)MK=RN8mhUb#QGpAjmab~T?u_Jk(BPvu=^_uuJDVZm02B%|3e7j&Zq6AAoY%y%Fr zj%b@4N_teudo{l5&TH#+;gp@jS0we~&~sHY9~Y%K8tTX9(Slcpg=%b5|Nj zbQ<1A*}2=x)91Afly)Jqil>-%D&dLe03{v0z$?3K0Nb~moIeN?a^@ICFuL~PL5}XG z4j!9^$z6HZ<$$}Q1kG&kETSaff}S`KIFbN@bTmXbiOQ6{nxPh zBL9;eSe{J+g375qyCw+4ard44Yas?>D}<)aOA%4<#dJusy^p!x=Q$@AUYBL+LmP2M zXJniLruLExg%bTj%>EsmWY@ionRWT%EdNvg-5uTLTAFsu9Rz>*S<$Abq*H{5!e`0g zuS$(uA9NYUpAjvn;yR*3{Bm2r!TKs}bINukl-Eg{+;S%rX=MC`T8rl=0HLpE`nH9F zpzWj~H2s-6{O~p%K^(Gjid<&Q1AKYPLnG|AdBYXvXNV|VdadjNl-zES_r5!k_9Zxw zjxD5kI(*B%4qBA=Z74a4urXXTwOaGLdEzLl6$gZP|6E)kc@Y)3lv`a8?B4Ig;^5la z$1#j0eHP3~hLsa>voo}`WzyjQn)t5=E1jF&ztUl*_8FBBhVum92_pjLpGpa!sIPV7 z|0Q!#f8wOSBz=?}r0sD2>)kH-cZND&b&_1Lk%xiYd02Q#Pj0*FfA%9e)?qs+PB@Hw zQb|7-47PH?`(w24<-(HZ;P7?^n5p})g~_>Q174_nR|ysw;L05mZB)|Dy&hndmk4C0 z>+X@kbw9xLjSNa*_5hBrE#eX3{^*61xVe2{T^GTz=eHwDGum;XrE_O7x)so~_XWor z!m{jw4?pGB9y=(uw$j$D5Tx?amJ~t*jr!Xh2LD0#S)ziGZPU0#ff3J#m(c!yK_?`g z+SL@O1Jhoi0GX!he+!GYyi4D-vnON_#0$SbOCf2#j1aZWDYPg<(?VlMH|X+O2G!B( z+}LdB;deHNd97JIm-x0B@Jrypz8vPqUG5UUj9d378UMob)m`aKRfYraE-wz{+VT)#*KLNeDq(tlfZEvuH8uolP~XU75BtXW}VZjzEH^pXq%Qk z?{sNez?E1c1H?!bwIg7S9U#>Dye8wBGrJVGJ{FSDMw<1_UMurTI&>-bdBsxb^o$4J zLJ5~0rhk6%WmvN@!W^WI{3G~bX9a$ctWgLOI0Vlv3?*W0UsBN1)c(3v?#@f9peK+i zav}I2_DV2HI%7E)-GLHZj`GXdiv{59J|x2Zl7wpd6kiYh2Y?X0)8KYRP0`?V=kFy+ zf`Gg8;UDf1jHr=Jz9$n)iNN1+DR|+f{C(H2QOle1j;78P60Qw7UMj_UPp_U7Lmg(x zs!SEx&Yx0NyCuMqOX0Y+jnWX1<;?;AlG9F>rHm|>&D>@+b!cLSpw44!)1^Bi>P(CXfq>E-;$`^9uv+4`smPz@1PB49t z2=!&+=^pWI4S*!5L3+!-Ib61#-1u(`SJ~kN5k*UZk)g?%THtaRqQGHn)3^M)KFXy(CjX!E*-9Zqq3Ke*92=WOxgC zV%H8j!Z#k|x#K6qTS50Tjpv~RuP1fff)hDNxo3lxLc5TrFGM1|0|s!~nK>E}F)RO^ zXiz9K^I?kRUK&WP;l|&@8(P2402R2Ew$le_s2vsEtY613)z61NQNN&pa5Ax8H9n?e z`aU^@ZgjUlW~sLe`(Bd*UIQQn4uTVg%mI*|1p9F8n-Q%St(TZn?4ujzP=&qe5a+qV z$CbgFcp_75S&G5-H{SjQ(*QwP%t}NBnm~uwwoKWQt+%Oq|J!=M3sB$lEMz7l*ZvF- zVtkG+)Az{STcNRjxvl*X!VFowe!GNM{5=zjY`U48(O40%U>GELbd2?o&5RLZc2mMa zLWAWg+doRK0AuBo1A;Mms9gCRf9vyAIxOG*OzL}_-u@Kh5bW!`&*Je>`Iy|kO}kEs z?ej{EjVX@YG>j9lm)Q})Oy-tA;!mQa{>?0~Z!8lVM)?U}O{)F5L(})NoB_G2_nSR< zKVJ>_rv`b#DtPY=fSi)Pb@V-4`(J_t{Y+(cXsB*~ZR?JQOidb*^g-Y|FH)#j^`MCd z#+k6kiXwFP{**#M`wg=~R}k7{@ehcPiGdLW^5*c#Z~=ilJbnr}>-z8#<88bR4tVa? z7HU0zyN+<&xM?h;cvgAK3K}|Z=tA9;IKjp6Vl7;@a4fEoPz zVbcJ1=5zNv7&>SbQ*RQwvti9ua)CW0I^i=R0rF&_eEaMV8HdLi<4^Mnel4h=Rf_oH zkOLW5-HQ507D~baB?gKhs_yXD>u!jlJ z+mD>vB#=8H$b8@D2XkG(huX1$|C;tt){nx|#e@%uH!1)942y!#G7rcZbB!S+T9I#($)9!(RY)wEf?^F z{5TWv5>J|6=QC~i>QFDf^`iPo14+BS-5;QrEnNHx_>hqw>!v)qeTT5t4^8-FbKB=Y z{yM$2KoYQ=79o9jM-oqvQo~?@F`D{khy!s;`NEg}qV)(rn43|-oE0C4mcS|ovjM@- z$yOj@tK_6D8WEAcof+9eQ4 zlV((38( za4Pjaxi9l^9q_t*4Y@t+vzpCU9o?XYIAwRYeM%)sfBskJ2}r^RNh)O9ap8^BR_S;J zQaGfAIve6eEJC_`CQq}2mvw!$-<{y_pH~^qemw4dpFKq*W63s~pzROrJSev7dN1ft zJmqK5&l?`r_VEC6|ISY=nxs#VkEy}I+iZX@U;=XB_6m7iSzEA@a@+)IDJ0xf;N-8M zX?Y6zyLKK9b0wA3*>g}; z1ksQ{|6ZV%I+j(Wt7>+${PtbL!ptM!uHq)7l`M&_D#!6E6pyp zdNLaBJmivbu#zFe0cZo{fIyT82*^xHC0W5wNa?-=wBATeb~>p4b0Q>bozw&loCaxSUw(GF`?zJHPr zY2I^9I}W*)(QPlxY((KlSkfmy2*Tap2*AH7m^l+Ah3!{JYejEdeq=Ig*VXSMizw~s z0IQpSvqon6Px2bO_R2oF9;*gUNO95b{+?C0di6m(mMo;?`Y$H(#rCeVI^ew`#0w1g zgY5PJfA7F|kpD;sql{*5<;7y|-%aidz}m5YOF)r=?m@->g2d(S``Hw7$K&=9#jPvt zGu1m?9W!4xQ8u2eONpd@NW2Wf3B)iDWC8Bd+Gq5EPbZ5rogvy4mi~()Hu7NJ+*#g3rjXR zjlBLBKa%xC;4qM?*mtAmXN-`(+5gdaV{xYqU{>Vn0N9x&- z#tUZ47*1c}(PV0`&Sx@Bbl*0BWiQFF28D31W`F*+=f+BS*WL`7PHU+~N-)3lK2r=) zk4u@7p`QhNvx5OBpUY8qGeOVa-1tRKPk(*Vr|-eGFV+3-04;h0e)2ms^W(rXcs@Gt zVj{IIs%sCt0l$)qeB)Ubv{4B`hXNUf;%yq?`|VNaVn~nRvmb$kd&b~AxNPU{>K?>} zcZgsgrQ=&2{2O@KT{_k3Gzm^r!!&V}Nlfee_S9GuEejmfk-n+s1rkMqc-S9h{^caNbxXrNd5bW9Zph z{X|Z9Q)|ku@BadY#7h7ufJyhUREb$8LYTD%5Z;V(aOAQ-kAwC)zDT%PefPe9Kb-^` z{0$k3)VnYQdG0o>K-tN$nS+efe3|4WI?Th3A!?@*0vtwoicgq2mEwP#JNNqCClP#y z+V}3iI2pVb&itm_`J6nN@@oBfS|;duPQS4+ulIcfsmx$orrlK#YQTR}BV^j~7nC3t zj0x{IDIf%}ILL>9MGEfMXYUaqrSrNRDy;sz(|+8i|NeUJB=AoNI60^uJytFFrn&Jx z@ErcG)*F=61#rGvy=!ZL#D&L)I?5g<&@!h00nFy6r6G;czx&TgCV%ta{w-hiyLmD% zWnhKzzfbS&ycs|F*_+1@_?h(od8fedzLf6sR^JzC#qfkbmCw5v`$;U{TC>ipJo3lf z$^0<_;oh>x6%MpB(r<| zs3u?QO-Jf)09V%FEd#Y)vfj?6xJ4*!cJBY`7`Zw~nDzhcZJi9e+397qK$iC|ko@KMjklD+%cjBGi^#Z1 zZ8HDYYQip`BLyCQ!8wn>VCEpu@7^YMIBw^3l(%+z*L*%Z+$54u9HxuSvgaOp#!+Q zSG(y<4`9jDgxzchi%*}0uNAiy|9fwU@DX=RJ)ZB$)CnD9moij=)I8{4zqiIh_tm}RmUE3 zQGBt}vvZ7XQ_Q&Yhe)#b?am+6_<4Fgl%d{RYkcf7E!6S>LBI3-!Qb=~0}kkSgLjeu zaL4|QKO!u-+s`))alwrMGo??!Pit_2Ha9e1!p=9=oPs@Bl0xv$I3VPsflRCy92sQ# zR%nYW!6O&LLxCSp=1&8pw9_;TN&3Sml2T_AQrl^S@c@oASYAcC4>w3(SyJh}VWayy z-*_2iHfKxggLToOK4Z4kQ~ED+Z6;c#Q;2(#-| z^hof_vw14pj5Xw~W4+Fx63d3*m*B&rb>Lj_{-_I*~wSRp_2CKSFbPYfnt5G$2< zKUjU1?!(1dU8BRzXeSc+2o5oCb<;Fx=cDOZxmG+fChzNyBnvtFMjgCOcJiE-xOafb z3~&6F8tJN$m28i9eQ|6DAb-m+=_EW0R;_MJ=2{f&)cg1>JJUR|nOWzWd~XpWS;1#I z`HIku{`jhmX9Hg>>p|^=6qG67@FIwMGR)xn*4DGnPP`t#`Z% z2vt7st`|!fr01&SSYt<7*o(hz);7&hy)q(b1gPjq>V=s~bxIE4CTR$FO!wQiq`;l2 z)5`p&%fNL7BZX)tB&RJm&R!m`+a_b7tqyU`)_Ee?<~}Am@#Ic7)6xF}p*?~=rvey& z-dav)moejL>2rBCCc?~*qs#l7STb(KY8=(nU*zRlGs#-GF4kZY4Le9|evFQ+QrefH zXqsK6GGS9Q(_~B^LfJ5kD%p@gp`*oYHA|!BFdc8Lbhdm*#@q5osYYp(Ws0yS7_Wk&8iXfs^`*brFzx^5HRikC#|)9#>L|=e}j0DLq^OO?d)hf8L{bpi!Kq&6=I6 zGa?EwO4W{_X_ZhfC4^Oph`)jrr&_{{O?IUhyQ?-buzD)xT{V|3tVf&k(Yvth246Zg zJ8Q{7VBHYX5Yd)s);2gzv<$7;+oEEJlmrb1KJFqNG~!#`0%R8HPUZ@el9@I zY%xO!|7eq%4g%JerKs)=%TtV?rK4$-%>;_)FN#lJ#=jR-(Z&rMJ!x~Y^`#p+_)n5mo&7*@Yr)t}$l|QRTr~|Uta)1RO*5*|nW-97^`WDAwf2mR>BDrJW9ygSje(Eu z+ZrwAQ#Q1dBG52FiKmn0^C&l9n>{Im5tl9>g$YxTJkKg~L=r zXSrOoSMfTB9K!fo8rGc>CQAmT{UR6e8;~2RGL;u-thJ`(7s*(1!_(PQ$Y4 zICVP;QGLa(`UbI~W2>F~wO%slS7qV45=7M0EuH5Tv8veW?ueqQ&)#%r-%W#PGXMgD zh5?aobrj8@eAdY_ade%Th_Tj~*ph+R+cvE4>`cj=# z>oLS-7V0cwH`K>GWXO`ArIkqyIEh<{{8f6N;#dOE>dFeHE%S0S6 zP3%~=O?)Jqg{HNegQ{^%CGwyK2WMXEH*?XUQC220D`td$FcS~8a(+Rgjq1h-9w81H z*#$UQ^xA6i}J~v>8NE4nV;8R6{ns%nCtw*fR3bj*+mxFlsHz{W|G*~Lx5t@&?l+{tk zlz1yNuR8!}w)D&w^`XV#Fb&G()OuJ4r0ty}?J@ObEd9*c^TA3ZOee)tT`FZRI2Y>S zW$DtRRcT~U#E2ZYiVJJ|E5@Zw&(zR^(CaqwHS=tG;o0ay?}f6fCygme?nKn8{YzA3 z! z4B@JrygPtRUr454{Q!p8`d`o@JvFS8c#~*Z?E1@T&GS_hnFEH^s~7vt`SZsz<^f{! zTtn>k=-;FRt!eL|I;Pj2-F* zGs&Dai0~SJfvAh;YMKcfUx9#z2&1dTql5T$RN9ml+p&0Lj~r_@abBsMsobeSPTVU< zT;qk+-H7z?Dn!F?&H{tbj;FeqTC8AJ@+IiRkKTV3C*&-mSuyDI*h2^Uu zovE<67juz0O(23_ma=wL7qp~k2L$x(Wt>4jpe@^%4rc`CU2&hMgsh`Fg3;np@~1tB zNR_vF*biZg^YiH|QiMPpdz*su8CrGCp zLhIYwPCze!)97E)M3?ZxdBA5|UvN>|!H*sEdu2~f)$AUGF^g1s`W-+1m5Nw#gvI`yEk7n_lGXIXZ(jv1%%o=x`^(k zN<%YVe2{?wHXuQY*+0G@ODDyT(zr>_ zW#aCQCm0Ske>ho;vXe)jwg5JdiHzqa+$?j!3ABdFwro~nIczAqE>1`?16326*ct*I{UA-z2UcViY9pAH>UgD9({d$^*x;__yjC`M`&Ru2DZu$+neeip!I8gwKwjJN z=r?jooph~t(CSmmrap$zmC(2**RE2qfJY8|F_Nqc7YbT6F$;k^4fmfKQPM8drotZl zB5O2wI`ubO+az6a@>uES?pOho!fJ^{y10l?TMpu+IF!zrC9$#ejdE};R`WG;;F#1~ z;(?cmAl46#O7%D*b&PNAhb?NY(HhVX(#*Km=RqAW#J*jIOH?d77fi5%rA+{3)nWP&TXc;j*+$YZWq zQZCICpbUTKRE!0fKIc(q^mu$u;fBwkcZvoqt`#m5$|z} z+2s!Qb02+NEKzvj%5(WBIya*^tH)Rwd@ zBh)*^N9N%s>Ib+QP*L3_*-t^btc`$@#F{`CBF{xYQ+`V9r)c_gJ#LZ zP!@p}Hks1p_*m@5YE`=3RFs3)Dbgl5IEX^AQVDquD-?)6EbFc^oln8c=L(0w0&rE) zXncuxWFg;K%9+!o- z&g>cF@<0^@6kSCxTeJf)DI^S>n>Kh>p-C~mJo87!7I=RpEO8tY+H$sywr91QjMG8pCT zJk2Q#thsIA3zb~4>@s4pP%^j@wy5R*2R(4bo)Uc{p@rog+{jp~4FOL3_*h|3*OS}0 zzmj2Tz`h4dW!{~I?Bj+GqbtIJ6R&MF^d2uJWDGy=r`T6kKCckJ_G;;6Hr&33tawbUU)IzFUJwv*QH`rROcZflOSVphB4v7P#}O5G)3vk`Ud0EL zLLj`hAc4Kz%ss&#ft2EKpMwAd`gjrx0So&jR zxaF1-6@*1#6I(@;{RKyNw*a*#|If3%!SHZ}@XXS1!bdpsuLA`Jfv+qs-viz@Cjy_DL8?g! zg7&7R+S*@^dno+AAJ0h$KM(!bm-*4Hfw`C9xS#@m2Y}>f?SpzBzG0AtI=mr2n*aFu za$@w?Vx;pxWNaM@{ngF=rs6pl$i??-$5`Q7@14tFDf8Fz)-H&s_fsVI$xoWIM`4doS68y@gGI{aTECQ zwM?Ci{-_)Y8nidm`u%*q-ru{_K(8g{5cHmq_1ykG8U4*5U=Zs3o|rr9f$BAz{vdq|=urH&>^mI5+ zq<>-9ZT9nfO;{hp8RVzRTd!iXN7C@1+in{F_^j=RuiwGVZ=0{JolWmr!JW&SlxW}$ zNnso*3lb<>()`#WhG(b$P12@>5cb45apeDNU!XI=<<(ilXhwJ~H-W51dATv-W%Nd$ zInvnO)9e*%hI99dgSaQoRVcT`&cX3ya{DOT&(XQhRm0~I^=X1*i4F1|lKIs&hRWFk zT(UhACAJH!FzOe;JUyKfzIN0vKEG`jWa^N86v53@L3Pwk#9@zE1uGsvKC1w7@iskq z5E?8Jb{9>-4{18sF14(d+WV9I(*=2w9JaXg5k3(>p(73k-*gs}$!ugdI8SR*9c~_Y z9b-YDcr{P@#5`w80KoDf)9Hbsd6WDh-j}UMSokZm3d1l-V~>1ms@}vXj0qq;nJ#yd zmx14|QZ%6ER%F+xp`Y6t9Hi==Tys$6=n-}g$E|kD07vN#y0`>oG{-i-f#_^1g`E)E zr~Hkv{0cs!H|#qWP-SXb@G1x|q~EKhw8Py&(wnPSe__wea*qLsZz~sCgEoZT8UIOd z=f0kBN5fp3O%R=K0(5y9+VC(z z^Dcer9s@yF=16df?Dc$7U5;xNfPd~WwS_`{bp3q1IKOH6{JQp;7;3>BEjhU$bdR%c z74C+3NWPhFTjTFC35Qz`?@yo3Kx)yy?>uBMb|@kvFt^FVcA{RsY})J-c5V`yr=E@n z{FyY*YC79xCRmgvakvh9YYk1mFV*~1!`N&K#h*9E5Rt_tF^P;qLXiFs1Qkad5vN9L z(i^w`3m?^N$+{80!3m&4vuw}p5dA0zFD8yvz`5-pY4#O=ba2mKvY~oYd3&t5ieXUh zcDj8^L1g3uypV+0XT2tzeSKP+gb>Frcu$7YZu-8_{Bqc6bz!IoXinC4?W~TtDMrIH zV9lM+L%|Pa=ZD8f`HtYzeuZi1<4kr&HYccnEP^HvET$)?Ad^PDd+Ds4?q4jd^dN& z-NtF?wP?Sv`QdYR{J5Om2>B6@!a%|kc7dEH5)DWE@94q8>tOD(Ynx$C0xh^CCBw9s#CAa~4(p3>dt) zZ?ZvmAdgBR0PlfCVnP{`MPw10I`q#?|MaxGKc@Lna7H1+pSEk4`o6SwMzID^&49@N zMSXC0Na}}-5MD22YN^K3Ww_dOm{1{*_eE;iNz_8TWM+4-isF={QO!T4N!-z?en&Fc zCi`2281ev+e>u=b@qmh+jTx!x4wMP($W(2mnuLrW^TC5JY3P1t32t$Q}skx@O;Qd;(iNNQM^%7nS@r_ zrXSBvy88)Sr>|L<`6Zh!wGB<)F6DycXuaMi)S|X8SudU$EB@0DZ|Sf=6+Qf)m&3!U z^y!q8_1%S_F1px(`oV^h`m-0UwYZI8E2ctxgV@Z+fMZyUoT+ART7e9=Gokf}bS^y= zYLrf#I+l(m7-0hIiOet!bo3B4uo*%hH(0Z>`No#^%B5tcJ@RYu7{)DPA?LO;R@SUj z2~zxYFeSDvO{WbHt5W#k#d5gHX=p$e#~$7X3cSO}HBq`p!%mm3dOWLPjv z8uq_sJ4?j8^zJO%=wa3ztUY3@DsczPbD2`YaO^tIX)YCGU+hA(vcr~kRu)pm+BbnY zYIsQdoj2J!%PBe6RV;1!@H4x5phq(4b=Txv*N~iWRum~A6b@!$(wkQbuZ>$;yo+l# z&~R2l?3AzjPCmse8AstVa>9K*)$Sx)RfDQKb_lPK%<7n9V}+#d{(KjhKXST8xVBcK ze|Wza@mMW1&?HsqV{a@tK6sumUvylT$$Wn$568|Xll6@cgY_L5`i4DOn75^*yyqa8 zs@xq(*>)}b^E;A*D)(8|7w!$Jo{I*)xdEie*AYP0wxXiht9GfDrEg8{wyBa3+}JO) zzpT}Eig{DrWC(T8GEtd;69n0J9-IW^r5Wz&^{3tUZw#85mk7t2tEtCn%o2BM3oQx_IOSh*c7WKz`Q z!oW>S7JC^xW>y1FhoqLCv*=X^D6qPLVh!qh^c^=6X(v_uinxoHA%stby_c5 z?BWwGX*>9*)726-<233EmrBY$+Ao2SCFJGwjvctoY$@9dnPOuxTh>wEm8WwDV;&6U zmaVQLS{r8!HVu9@rAhTAx;`~jHtWXQ%@=OmKJ%}(9(&v8;<^e+J3+Ev>Z+ILq%o~V zvg$l%fc}WYUb8T{8JR`6(bhfDmKOR^lq?EQQwesXPafrS=DJ~(J-4AL9L$??7`3bV zA<1%5S#$dNEK?{k;YI8iWqdv!VigCno&XvjNZ6hWi0UH6!A+z=HnS8;9%ne3CASk(4X~lETy{)zvVB?Zy?nKgn$t&CJC+2Qk-g2pjoNvRp_$aEXg8GQ z_72a9o3!*%K|3jB`D!9FD;hS|W@iSjJo8DlQEaA1KZ1_o$mkw9y4hK-&$9Kymk+uH zXJ8t%Zte}OGw#F>&$bEOxT~TLPm2c+3xQdt(ue$#erU!gSmI%mor7xo;-je?io zqZt>8NrjgkC_!CJ;wj}uzk)WC9L_E3xD|giM|;Nl^%gAG9}JjtnU(7txEI=PSB{=|t$={seZX zJ9mfXHI)(ZF~B|I&9AYQD=%QfZdlEa?uPVNMxs>6VRwt_%WPf~%Q6|4DqW29Zgw`5 zj5yNyvP0k2oJZ#p+(ek8TA$oZQNQ!YhL>BEoyj?@$66UL4rtTa+#tWI>T=>f_4YG1E zYRAT_n49XaIj)xn`=3rlhrbqccIer3J8YeA89eG}Jk+?W zGj8n_ap~hdMKyIhty=Hy`|^nOpf0(11&yuwq%b_zso7U$a`-2<&zu(T(Rn|Fy(v zbK^oD`m9zNIxo~LV98LSp|Xz9D6LR&Xs+H0v7l?Dc6G# zNoTglZuxJKJ)8EEW{NATs;`P@hp=F!x&hbsdz2kbO}X^#&Opx<*Pn^us7J7z_tSvz zsv?Q0;Yf)ybr1?C&phv6SW1}j0CZkZx<4vy8YqJcLXIl)0fA_#4b$Eism#G8>atvKB!tkXi!OR zjB#PkI&#R^)or&#uP3CEY8p?Ww2-WHhK;GhTmgN-$Pe*kJkBQNB=^jm94dvGnsC~Jk#Tuzbb^*_0%QB-Ki$ArWo zH>Rg?{zJNw7}?k-C5)Oz4;I*-X_euTQ}{zd(^^DOVCHPvGYmy<_z8$65DvoSGwMwa zGP4p(9r%PrBu{hJ?b4*4T9Y0$8!%IxLKURCCmWlx&wWu~uuJ*+u4!29_MsQVrvfwO z=N_BzThww^d&eHcdN{SAIS`s=jlZIz$Lm6RhE3Vjg)&HD=7qZDU)5!SEJ5QuJV4Ac|P|!|VLd#?tNhKqzWjv91O9AEo@9ae1zHI4_keVwxfs_3wJ6Pto z(}H$5pB=VAQw4*{F0Uz!+U)u*AVWj=Z&DGMpVgEZI~PepXIs9^@e!7|OvP!$%iDzL z1)#jtm`ZeD6am)6#lpx1#Xa=m)I~9z<+9A|Jd-91x6@e%zXqH5uPqq~8AkbwyL-CJ zcW7;X*YP8I!DpH=isS*g;A$<3cs420NmQZwz8IFd{Y~au`4R@Tyok6V#BfT<50q6e zeqH}e`weV857Yf@J1W_wvjo1^#AxO)Tc)Y}-XPxemogPwn5bWh2!|EtNA9lCzTyNX3KLsHdJFL0_5}-lNR8c$& zhJzJ`tC(0V1@NcUO-s+IN~#0Q^YEIemmcYD63Qsn6)O5o~;Ea}& zd-`(26pHn1{bm91e=v_Ca6}c~?&bzmcwM6yN7V-pd^$KI{$@!t+J5yJ`l-vX0SC1Z z2M3y6`UGtDyc>1BhJ_;`Jg!-!XLR1pqeu!35eyybV^$ZXrHHp{#z4WSnx0mmLxbKQ z!g*V)g|)_{1jh5kO=!xzfqY6IV-?OLlY@QK307J|?tllik8Uken^4aSA<+2|7`xYW za4+mpdIkpT*~?juML~7l^g$JkegsP4E!kx*-@Ry&aldN5YeM@kCFXc=P)-x%u~pv4 zSbuA~Db*TpbPHsXTS^Zl=~^C1Lui@`9pBHBF1(6Ph z&~jv-e2& zbQlKb`u&Gewy;i4$RpRIG`?s^2pEAUOq_vLb)ZrW)oD!>pq1zD+5GA;ZyN{<3(fAi zJ01c@#4yAH*K#+00d9nYP8okf76Bn5NTWYhYo^?EOidl>Mdt~Tub5?5zyuaN(Ic`M zooCIY_e&UABL1X2&)(ccJbI3A+A$48B>scsFoP8P6JYyVP>~mELVB%e28-CC;t-65 zlBW2rSrG^uOD)zxJ4`y~E>52v%WG)ilG;LLV^`omKUJr5nf6Bm>Y1qD zKi=t^R}TuQNO$ZTWs$Z(tC#0<2H)3^26}5gqmH4pQPyNKj-aYkftj89f%0 zau$uG9HCo$8j-Rt`PrlDKtN+Zp2eF5VLr);%X7Y=4D1bo%ZDHbWP~gEsvkgVz^&Gb z$5S<|XL_5kAQFEyxBnrSBg4?}{_@%N67HeCm55;jeqnkR^Wcm?5IBZCh&xKg{E=BW zR3-F#%6CP#Xn?!W>oT-Li$Nn}Si4=H4-E9b054xB-~a#s diff --git a/apps/emqx_resource/include/emqx_resource.hrl b/apps/emqx_resource/include/emqx_resource.hrl index 41e5673d9..10dc001c2 100644 --- a/apps/emqx_resource/include/emqx_resource.hrl +++ b/apps/emqx_resource/include/emqx_resource.hrl @@ -114,7 +114,6 @@ %% boolean -define(START_AFTER_CREATED, true). --define(START_AFTER_CREATED_RAW, <<"true">>). -define(TEST_ID_PREFIX, "_probe_:"). -define(RES_METRICS, resource_metrics). diff --git a/apps/emqx_resource/src/emqx_resource.app.src b/apps/emqx_resource/src/emqx_resource.app.src index 57ab8129a..19b6ca5e2 100644 --- a/apps/emqx_resource/src/emqx_resource.app.src +++ b/apps/emqx_resource/src/emqx_resource.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_resource, [ {description, "Manager for all external resources"}, - {vsn, "0.1.19"}, + {vsn, "0.1.20"}, {registered, []}, {mod, {emqx_resource_app, []}}, {applications, [ diff --git a/apps/emqx_resource/src/schema/emqx_resource_schema.erl b/apps/emqx_resource/src/schema/emqx_resource_schema.erl index 59687eb8d..b98f50a98 100644 --- a/apps/emqx_resource/src/schema/emqx_resource_schema.erl +++ b/apps/emqx_resource/src/schema/emqx_resource_schema.erl @@ -129,7 +129,7 @@ health_check_interval_range(HealthCheckInterval) -> start_after_created(type) -> boolean(); start_after_created(desc) -> ?DESC("start_after_created"); -start_after_created(default) -> ?START_AFTER_CREATED_RAW; +start_after_created(default) -> ?START_AFTER_CREATED; start_after_created(required) -> false; start_after_created(_) -> undefined. From 8cee75139dc2d15c5007deb137c67233696a3b10 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 7 Jul 2023 18:06:28 +0800 Subject: [PATCH 49/92] fix(bridge): ensure the username of pgsql must exists --- .../src/emqx_connector_pgsql.erl | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/apps/emqx_connector/src/emqx_connector_pgsql.erl b/apps/emqx_connector/src/emqx_connector_pgsql.erl index 71d18f4a8..c468aa8bd 100644 --- a/apps/emqx_connector/src/emqx_connector_pgsql.erl +++ b/apps/emqx_connector/src/emqx_connector_pgsql.erl @@ -69,7 +69,7 @@ roots() -> fields(config) -> [{server, server()}] ++ - emqx_connector_schema_lib:relational_db_fields() ++ + adjust_fields(emqx_connector_schema_lib:relational_db_fields()) ++ emqx_connector_schema_lib:ssl_fields() ++ emqx_connector_schema_lib:prepare_statement_fields(). @@ -77,6 +77,22 @@ server() -> Meta = #{desc => ?DESC("server")}, emqx_schema:servers_sc(Meta, ?PGSQL_HOST_OPTIONS). +adjust_fields(Fields) -> + lists:map( + fun + ({username, OrigUsernameFn}) -> + {username, fun + (required) -> + true; + (Any) -> + OrigUsernameFn(Any) + end}; + (Field) -> + Field + end, + Fields + ). + %% =================================================================== callback_mode() -> always_sync. From d01eee7fe4ee358af4c301bfd17b5fd3a53214f1 Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 7 Jul 2023 18:11:52 +0800 Subject: [PATCH 50/92] chore: update changes && app version --- apps/emqx_connector/src/emqx_connector.app.src | 2 +- changes/ee/fix-11225.en.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 changes/ee/fix-11225.en.md diff --git a/apps/emqx_connector/src/emqx_connector.app.src b/apps/emqx_connector/src/emqx_connector.app.src index d268a244a..9dcec9187 100644 --- a/apps/emqx_connector/src/emqx_connector.app.src +++ b/apps/emqx_connector/src/emqx_connector.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_connector, [ {description, "EMQX Data Integration Connectors"}, - {vsn, "0.1.26"}, + {vsn, "0.1.27"}, {registered, []}, {mod, {emqx_connector_app, []}}, {applications, [ diff --git a/changes/ee/fix-11225.en.md b/changes/ee/fix-11225.en.md new file mode 100644 index 000000000..8d7ad554f --- /dev/null +++ b/changes/ee/fix-11225.en.md @@ -0,0 +1 @@ +Fix the `username` of PostgreSQL/Timescale/MatrixDB bridges could be empty From 97b6c430621884ccfc1da9bd02b9d6bbb24cfe4c Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Tue, 4 Jul 2023 18:15:37 +0200 Subject: [PATCH 51/92] fix(ft-test): use new cth tooling in `emqx_ft_storage_fs_SUITE` In attempt to battle test flakiness. --- apps/emqx/test/emqx_cth_cluster.erl | 14 ++++--- .../emqx_ft/test/emqx_ft_storage_fs_SUITE.erl | 40 +++++++++++++------ apps/emqx_ft/test/emqx_ft_test_helpers.erl | 17 -------- 3 files changed, 36 insertions(+), 35 deletions(-) diff --git a/apps/emqx/test/emqx_cth_cluster.erl b/apps/emqx/test/emqx_cth_cluster.erl index caae62f4a..5e8bd4103 100644 --- a/apps/emqx/test/emqx_cth_cluster.erl +++ b/apps/emqx/test/emqx_cth_cluster.erl @@ -206,11 +206,6 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) -> base_port := BasePort, work_dir := WorkDir } = Spec, - Listeners = [ - #{Type => #{default => #{bind => format("127.0.0.1:~p", [Port])}}} - || Type <- [tcp, ssl, ws, wss], - Port <- [listener_port(BasePort, Type)] - ], Cluster = case get_cluster_seeds(Spec) of [_ | _] = Seeds -> @@ -239,7 +234,7 @@ default_appspec(emqx_conf, Spec, _NodeSpecs) -> tcp_server_port => gen_rpc_port(BasePort), port_discovery => manual }, - listeners => lists:foldl(fun maps:merge/2, #{}, Listeners) + listeners => allocate_listener_ports([tcp, ssl, ws, wss], Spec) } }; default_appspec(_App, _, _) -> @@ -252,6 +247,13 @@ get_cluster_seeds(#{join_to := Node}) -> get_cluster_seeds(#{core_nodes := CoreNodes}) -> CoreNodes. +allocate_listener_port(Type, #{base_port := BasePort}) -> + Port = listener_port(BasePort, Type), + #{Type => #{default => #{bind => format("127.0.0.1:~p", [Port])}}}. + +allocate_listener_ports(Types, Spec) -> + lists:foldl(fun maps:merge/2, #{}, [allocate_listener_port(Type, Spec) || Type <- Types]). + start_node_init(Spec = #{name := Node}) -> Node = start_bare_node(Node, Spec), pong = net_adm:ping(Node), diff --git a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl index 50925cfb9..a57cdf621 100644 --- a/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_storage_fs_SUITE.erl @@ -35,10 +35,18 @@ groups() -> ]. init_per_suite(Config) -> - ok = emqx_common_test_helpers:start_apps([emqx_ft], emqx_ft_test_helpers:env_handler(Config)), - Config. -end_per_suite(_Config) -> - ok = emqx_common_test_helpers:stop_apps([emqx_ft]), + Storage = emqx_ft_test_helpers:local_storage(Config), + WorkDir = ?config(priv_dir, Config), + Apps = emqx_cth_suite:start( + [ + {emqx_ft, #{config => emqx_ft_test_helpers:config(Storage)}} + ], + #{work_dir => WorkDir} + ), + [{suite_apps, Apps} | Config]. + +end_per_suite(Config) -> + ok = emqx_cth_suite:stop(?config(suite_apps, Config)), ok. init_per_testcase(Case, Config) -> @@ -46,14 +54,25 @@ init_per_testcase(Case, Config) -> end_per_testcase(_Case, _Config) -> ok. -init_per_group(cluster, Config) -> - Node = emqx_ft_test_helpers:start_additional_node(Config, emqx_ft_storage_fs1), - [{additional_node, Node} | Config]; +init_per_group(Group = cluster, Config) -> + WorkDir = filename:join(?config(priv_dir, Config), Group), + Apps = [ + {emqx_conf, #{start => false}}, + {emqx_ft, "file_transfer { enable = true, storage.local { enable = true } }"} + ], + Nodes = emqx_cth_cluster:start( + [ + {emqx_ft_storage_fs1, #{apps => Apps, join_to => node()}}, + {emqx_ft_storage_fs2, #{apps => Apps, join_to => node()}} + ], + #{work_dir => WorkDir} + ), + [{cluster, Nodes} | Config]; init_per_group(_Group, Config) -> Config. end_per_group(cluster, Config) -> - ok = emqx_ft_test_helpers:stop_additional_node(?config(additional_node, Config)); + ok = emqx_cth_suite:stop(?config(cluster, Config)); end_per_group(_Group, _Config) -> ok. @@ -62,12 +81,9 @@ end_per_group(_Group, _Config) -> %%-------------------------------------------------------------------- t_multinode_exports(Config) -> - Node1 = ?config(additional_node, Config), + [Node1, Node2 | _] = ?config(cluster, Config), ok = emqx_ft_test_helpers:upload_file(<<"c/1">>, <<"f:1">>, "fn1", <<"data">>, Node1), - - Node2 = node(), ok = emqx_ft_test_helpers:upload_file(<<"c/2">>, <<"f:2">>, "fn2", <<"data">>, Node2), - ?assertMatch( [ #{transfer := {<<"c/1">>, <<"f:1">>}, name := "fn1"}, diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index fbb3e7d6f..448ece55a 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -24,23 +24,6 @@ -define(S3_HOST, <<"minio">>). -define(S3_PORT, 9000). -start_additional_node(Config, Name) -> - emqx_common_test_helpers:start_slave( - Name, - [ - {apps, [emqx_ft]}, - {join_to, node()}, - {configure_gen_rpc, true}, - {env_handler, env_handler(Config)} - ] - ). - -stop_additional_node(Node) -> - _ = rpc:call(Node, ekka, leave, []), - ok = rpc:call(Node, emqx_common_test_helpers, stop_apps, [[emqx_ft]]), - ok = emqx_common_test_helpers:stop_slave(Node), - ok. - env_handler(Config) -> fun (emqx_ft) -> From 0ef00d591920fbe40892aa756245ebe76cf80865 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 7 Jul 2023 17:55:08 +0800 Subject: [PATCH 52/92] fix: psk_authentication is updated failed --- apps/emqx_psk/include/emqx_psk.hrl | 4 +++ apps/emqx_psk/src/emqx_psk.app.src | 2 +- apps/emqx_psk/src/emqx_psk.erl | 37 ++++++++++++++++++++------- apps/emqx_psk/src/emqx_psk_app.erl | 1 + apps/emqx_psk/src/emqx_psk_schema.erl | 3 ++- apps/emqx_psk/test/emqx_psk_SUITE.erl | 9 +++++++ 6 files changed, 45 insertions(+), 11 deletions(-) diff --git a/apps/emqx_psk/include/emqx_psk.hrl b/apps/emqx_psk/include/emqx_psk.hrl index 700160a5f..ece0d500f 100644 --- a/apps/emqx_psk/include/emqx_psk.hrl +++ b/apps/emqx_psk/include/emqx_psk.hrl @@ -17,3 +17,7 @@ -define(TAB, emqx_psk). -define(PSK_SHARD, emqx_psk_shard). + +-define(PSK_KEY, psk_authentication). + +-define(DEFAULT_DELIMITER, <<":">>). diff --git a/apps/emqx_psk/src/emqx_psk.app.src b/apps/emqx_psk/src/emqx_psk.app.src index 26885673c..be24112e4 100644 --- a/apps/emqx_psk/src/emqx_psk.app.src +++ b/apps/emqx_psk/src/emqx_psk.app.src @@ -2,7 +2,7 @@ {application, emqx_psk, [ {description, "EMQX PSK"}, % strict semver, bump manually! - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {modules, []}, {registered, [emqx_psk_sup]}, {applications, [kernel, stdlib]}, diff --git a/apps/emqx_psk/src/emqx_psk.erl b/apps/emqx_psk/src/emqx_psk.erl index 6b2199832..7a0986fe7 100644 --- a/apps/emqx_psk/src/emqx_psk.erl +++ b/apps/emqx_psk/src/emqx_psk.erl @@ -27,7 +27,8 @@ load/0, unload/0, on_psk_lookup/2, - import/1 + import/1, + post_config_update/5 ]). -export([ @@ -68,13 +69,11 @@ -include("emqx_psk.hrl"). --define(DEFAULT_DELIMITER, <<":">>). - -define(CR, 13). -define(LF, 10). -ifdef(TEST). --export([call/1, trim_crlf/1]). +-export([call/1, trim_crlf/1, import_psks/3]). -endif. %%------------------------------------------------------------------------------ @@ -135,10 +134,6 @@ stop() -> import_config(#{<<"psk_authentication">> := PskConf}) -> case emqx_conf:update([psk_authentication], PskConf, #{override_to => cluster}) of {ok, _} -> - case get_config(enable) of - true -> load(); - false -> ok - end, {ok, #{root_key => psk_authentication, changed => []}}; Error -> {error, #{root_key => psk_authentication, reason => Error}} @@ -146,6 +141,16 @@ import_config(#{<<"psk_authentication">> := PskConf}) -> import_config(_RawConf) -> {ok, #{root_key => psk_authentication, changed => []}}. +post_config_update([?PSK_KEY], _Req, #{enable := Enable} = NewConf, _OldConf, _AppEnvs) -> + case Enable of + true -> + load(), + _ = gen_server:cast(?MODULE, {import_from_conf, NewConf}); + false -> + unload() + end, + ok. + %%------------------------------------------------------------------------------ %% gen_server callbacks %%------------------------------------------------------------------------------ @@ -169,6 +174,15 @@ handle_call(Req, _From, State) -> ?SLOG(info, #{msg => "unexpected_call_discarded", req => Req}), {reply, {error, unexpected}, State}. +handle_cast({import_from_conf, Conf}, State) -> + Separator = maps:get(separator, Conf, ?DEFAULT_DELIMITER), + ChunkSize = maps:get(chunk_size, Conf), + _ = + case maps:get(init_file, Conf, undefined) of + undefined -> ok; + InitFile -> import_psks(InitFile, Separator, ChunkSize) + end, + {noreply, State}; handle_cast(Req, State) -> ?SLOG(info, #{msg => "unexpected_cast_discarded", req => Req}), {noreply, State}. @@ -198,6 +212,11 @@ get_config(chunk_size) -> emqx_conf:get([psk_authentication, chunk_size]). import_psks(SrcFile) -> + Separator = get_config(separator), + ChunkSize = get_config(chunk_size), + import_psks(SrcFile, Separator, ChunkSize). + +import_psks(SrcFile, Separator, ChunkSize) -> case file:open(SrcFile, [read, raw, binary, read_ahead]) of {error, Reason} -> ?SLOG(error, #{ @@ -207,7 +226,7 @@ import_psks(SrcFile) -> }), {error, Reason}; {ok, Io} -> - try import_psks(Io, get_config(separator), get_config(chunk_size), 0) of + try import_psks(Io, Separator, ChunkSize, 0) of ok -> ok; {error, Reason} -> diff --git a/apps/emqx_psk/src/emqx_psk_app.erl b/apps/emqx_psk/src/emqx_psk_app.erl index f1a7cf18c..d4735f4c9 100644 --- a/apps/emqx_psk/src/emqx_psk_app.erl +++ b/apps/emqx_psk/src/emqx_psk_app.erl @@ -27,6 +27,7 @@ start(_Type, _Args) -> ok = mria:wait_for_tables([?TAB]), + emqx_conf:add_handler([?PSK_KEY], emqx_psk), {ok, Sup} = emqx_psk_sup:start_link(), {ok, Sup}. diff --git a/apps/emqx_psk/src/emqx_psk_schema.erl b/apps/emqx_psk/src/emqx_psk_schema.erl index 45a1a077e..e6c922c1e 100644 --- a/apps/emqx_psk/src/emqx_psk_schema.erl +++ b/apps/emqx_psk/src/emqx_psk_schema.erl @@ -20,6 +20,7 @@ -include_lib("typerefl/include/types.hrl"). -include_lib("hocon/include/hoconsc.hrl"). +-include("emqx_psk.hrl"). -export([ namespace/0, @@ -52,7 +53,7 @@ fields() -> })}, {separator, ?HOCON(binary(), #{ - default => <<":">>, + default => ?DEFAULT_DELIMITER, desc => ?DESC(separator) })}, {chunk_size, diff --git a/apps/emqx_psk/test/emqx_psk_SUITE.erl b/apps/emqx_psk/test/emqx_psk_SUITE.erl index 00702efa0..2a28ceb2c 100644 --- a/apps/emqx_psk/test/emqx_psk_SUITE.erl +++ b/apps/emqx_psk/test/emqx_psk_SUITE.erl @@ -20,6 +20,7 @@ -include_lib("common_test/include/ct.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include("emqx_psk.hrl"). -define(CR, 13). -define(LF, 10). @@ -124,7 +125,15 @@ t_load_unload(_) -> t_import(_) -> Init = emqx_conf:get([psk_authentication, init_file], undefined), + Separator = emqx_conf:get([psk_authentication, separator], ?DEFAULT_DELIMITER), + ChunkSize = emqx_conf:get([psk_authentication, chunk_size], 50), ?assertEqual(ok, emqx_psk:import(Init)), + Keys0 = lists:sort(mnesia:dirty_all_keys(emqx_psk)), + ?assert(length(Keys0) > 0), + {atomic, ok} = mnesia:clear_table(emqx_psk), + ok = emqx_psk:import_psks(Init, Separator, ChunkSize), + Keys1 = lists:sort(mnesia:dirty_all_keys(emqx_psk)), + ?assertEqual(Keys0, Keys1), ?assertMatch({error, _}, emqx_psk:import("~/_none_")), ok. From 9bb159cf1ec2b164fed8e6a7e26aa3bd25ce69f3 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Wed, 5 Jul 2023 12:48:09 +0300 Subject: [PATCH 53/92] fix(rebalance): fix changelog and version --- apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src | 2 +- changes/ee/fix-11198.en.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changes/ee/fix-11198.en.md diff --git a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src index c175097e5..edfa6574e 100644 --- a/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src +++ b/apps/emqx_node_rebalance/src/emqx_node_rebalance.app.src @@ -1,6 +1,6 @@ {application, emqx_node_rebalance, [ {description, "EMQX Node Rebalance"}, - {vsn, "5.0.3"}, + {vsn, "5.0.4"}, {registered, [ emqx_node_rebalance_sup, emqx_node_rebalance, diff --git a/changes/ee/fix-11198.en.md b/changes/ee/fix-11198.en.md new file mode 100644 index 000000000..60173dc73 --- /dev/null +++ b/changes/ee/fix-11198.en.md @@ -0,0 +1,2 @@ +Fix global rebalance status evaluation on replicant nodes. +Previously, `/api/v5/load_rebalance/global_status` API method could return incomplete results if handled by a replicant node. From aba52c6a257ff9c75015fc3be6a67662072f151a Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 7 Jul 2023 17:57:52 +0300 Subject: [PATCH 54/92] chore(rebalance test): fix API tests --- .../test/emqx_node_rebalance_api_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl index bb691a754..188e6bf71 100644 --- a/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl +++ b/apps/emqx_node_rebalance/test/emqx_node_rebalance_api_SUITE.erl @@ -192,7 +192,7 @@ t_start_stop_evacuation(Config) -> [{DonorNode, _}, {RecipientNode, _}] = ?config(cluster_nodes, Config), StartOpts = maps:merge( - emqx_node_rebalance_api:rebalance_evacuation_example(), + maps:get(evacuation, emqx_node_rebalance_api:rebalance_evacuation_example()), #{migrate_to => [atom_to_binary(RecipientNode)]} ), @@ -295,7 +295,7 @@ t_start_stop_rebalance(Config) -> StartOpts = maps:without( [nodes], - emqx_node_rebalance_api:rebalance_example() + maps:get(rebalance, emqx_node_rebalance_api:rebalance_example()) ), ?assertMatch( From 27ab9c62d8f159c3727fbf816adedcc9c721079f Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 12 May 2023 14:36:35 +0300 Subject: [PATCH 55/92] fix(ft): unload conf hooks in `prep_stop` In order to avoid situations when the root supervisor is stopped already. --- apps/emqx_ft/src/emqx_ft_app.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_app.erl b/apps/emqx_ft/src/emqx_ft_app.erl index 299683e43..43a4cc816 100644 --- a/apps/emqx_ft/src/emqx_ft_app.erl +++ b/apps/emqx_ft/src/emqx_ft_app.erl @@ -18,13 +18,16 @@ -behaviour(application). --export([start/2, stop/1]). +-export([start/2, prep_stop/1, stop/1]). start(_StartType, _StartArgs) -> {ok, Sup} = emqx_ft_sup:start_link(), ok = emqx_ft_conf:load(), {ok, Sup}. -stop(_State) -> +prep_stop(State) -> ok = emqx_ft_conf:unload(), + State. + +stop(_State) -> ok. From 34793c5ed0aade552326a426e831ca587dc5314d Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 12 May 2023 14:45:22 +0300 Subject: [PATCH 56/92] feat(ft-conf): preprocess TLS configuration on updates Specifically, reify keys and certificates into files in the file system and update the configuration to point to those files. --- apps/emqx_ft/src/emqx_ft_conf.erl | 85 +++++++++--- apps/emqx_ft/src/emqx_ft_storage.erl | 24 ++-- apps/emqx_ft/src/emqx_ft_storage_exporter.erl | 10 +- .../src/emqx_ft_storage_exporter_fs.erl | 6 +- .../src/emqx_ft_storage_exporter_s3.erl | 23 +++- apps/emqx_ft/src/emqx_ft_storage_fs.erl | 10 +- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 130 ++++++++++++++---- apps/emqx_ft/test/emqx_ft_test_helpers.erl | 10 ++ apps/emqx_s3/src/emqx_s3.erl | 37 +++++ 9 files changed, 261 insertions(+), 74 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 2e994925c..0907ffd09 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -34,7 +34,9 @@ %% Load/Unload -export([ load/0, - unload/0 + unload/0, + get/0, + update/1 ]). %% callbacks for emqx_config_handler @@ -43,6 +45,8 @@ post_config_update/5 ]). +-type update_request() :: emqx_config:config(). + -type milliseconds() :: non_neg_integer(). -type seconds() :: non_neg_integer(). @@ -95,49 +99,96 @@ load() -> -spec unload() -> ok. unload() -> - ok = stop(), - emqx_conf:remove_handler([file_transfer]). + ok = emqx_conf:remove_handler([file_transfer]), + maybe_stop(). + +-spec get() -> emqx_config:config(). +get() -> + emqx_config:get([file_transfer]). + +-spec update(emqx_config:config()) -> {ok, emqx_config:update_result()} | {error, term()}. +update(Config) -> + emqx_conf:update([file_transfer], Config, #{override_to => cluster}). %%-------------------------------------------------------------------- %% emqx_config_handler callbacks %%-------------------------------------------------------------------- --spec pre_config_update(list(atom()), emqx_config:update_request(), emqx_config:raw_config()) -> +-spec pre_config_update(list(atom()), update_request(), emqx_config:raw_config()) -> {ok, emqx_config:update_request()} | {error, term()}. -pre_config_update(_, Req, _Config) -> - {ok, Req}. +pre_config_update([file_transfer | _], NewConfig, OldConfig) -> + propagate_config_update( + fun emqx_ft_storage_exporter_s3:pre_config_update/3, + [<<"storage">>, <<"local">>, <<"exporter">>, <<"s3">>], + NewConfig, + OldConfig + ). -spec post_config_update( list(atom()), - emqx_config:update_request(), + update_request(), emqx_config:config(), emqx_config:config(), emqx_config:app_envs() ) -> ok | {ok, Result :: any()} | {error, Reason :: term()}. post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> - on_config_update(OldConfig, NewConfig). + PropResult = propagate_config_update( + fun emqx_ft_storage_exporter_s3:post_config_update/3, + [storage, local, exporter, s3], + NewConfig, + OldConfig + ), + case PropResult of + ok -> + on_config_update(OldConfig, NewConfig); + {error, Reason} -> + {error, Reason} + end. + +propagate_config_update(Fun, ConfKey, NewConfig, OldConfig) -> + NewSubConf = emqx_utils_maps:deep_get(ConfKey, NewConfig, undefined), + OldSubConf = emqx_utils_maps:deep_get(ConfKey, OldConfig, undefined), + case Fun(ConfKey, NewSubConf, OldSubConf) of + ok -> + ok; + {ok, undefined} -> + {ok, NewConfig}; + {ok, NewSubConfUpdate} -> + {ok, emqx_utils_maps:deep_put(ConfKey, NewConfig, NewSubConfUpdate)}; + {error, Reason} -> + {error, Reason} + end. on_config_update(#{enable := false}, #{enable := false}) -> ok; on_config_update(#{enable := true, storage := OldStorage}, #{enable := false}) -> - ok = emqx_ft_storage:on_config_update(OldStorage, undefined), - ok = emqx_ft:unhook(); + ok = stop(OldStorage); on_config_update(#{enable := false}, #{enable := true, storage := NewStorage}) -> - ok = emqx_ft_storage:on_config_update(undefined, NewStorage), - ok = emqx_ft:hook(); + ok = start(NewStorage); on_config_update(#{enable := true, storage := OldStorage}, #{enable := true, storage := NewStorage}) -> - ok = emqx_ft_storage:on_config_update(OldStorage, NewStorage). + ok = emqx_ft_storage:update_config(OldStorage, NewStorage). maybe_start() -> case emqx_config:get([file_transfer]) of #{enable := true, storage := Storage} -> - ok = emqx_ft_storage:on_config_update(undefined, Storage), - ok = emqx_ft:hook(); + start(Storage); _ -> ok end. -stop() -> +maybe_stop() -> + case emqx_config:get([file_transfer]) of + #{enable := true, storage := Storage} -> + stop(Storage); + _ -> + ok + end. + +start(Storage) -> + ok = emqx_ft_storage:update_config(undefined, Storage), + ok = emqx_ft:hook(). + +stop(Storage) -> ok = emqx_ft:unhook(), - ok = emqx_ft_storage:on_config_update(storage(), undefined). + ok = emqx_ft_storage:update_config(Storage, undefined). diff --git a/apps/emqx_ft/src/emqx_ft_storage.erl b/apps/emqx_ft/src/emqx_ft_storage.erl index e2980c920..2d068466c 100644 --- a/apps/emqx_ft/src/emqx_ft_storage.erl +++ b/apps/emqx_ft/src/emqx_ft_storage.erl @@ -16,6 +16,8 @@ -module(emqx_ft_storage). +-include_lib("emqx/include/types.hrl"). + -export( [ store_filemeta/2, @@ -29,7 +31,7 @@ with_storage_type/3, backend/0, - on_config_update/2 + update_config/2 ] ). @@ -94,10 +96,10 @@ -callback files(storage(), query(Cursor)) -> {ok, page(file_info(), Cursor)} | {error, term()}. --callback start(emqx_config:config()) -> any(). --callback stop(emqx_config:config()) -> any(). +-callback start(storage()) -> any(). +-callback stop(storage()) -> any(). --callback on_config_update(_OldConfig :: emqx_config:config(), _NewConfig :: emqx_config:config()) -> +-callback update_config(_OldConfig :: maybe(storage()), _NewConfig :: maybe(storage())) -> any(). %%-------------------------------------------------------------------- @@ -157,9 +159,9 @@ with_storage_type(Type, Fun, Args) -> backend() -> backend(emqx_ft_conf:storage()). --spec on_config_update(_Old :: emqx_maybe:t(config()), _New :: emqx_maybe:t(config())) -> +-spec update_config(_Old :: emqx_maybe:t(config()), _New :: emqx_maybe:t(config())) -> ok. -on_config_update(ConfigOld, ConfigNew) -> +update_config(ConfigOld, ConfigNew) -> on_backend_update( emqx_maybe:apply(fun backend/1, ConfigOld), emqx_maybe:apply(fun backend/1, ConfigNew) @@ -168,13 +170,13 @@ on_config_update(ConfigOld, ConfigNew) -> on_backend_update({Type, _} = Backend, {Type, _} = Backend) -> ok; on_backend_update({Type, StorageOld}, {Type, StorageNew}) -> - ok = (mod(Type)):on_config_update(StorageOld, StorageNew); + ok = (mod(Type)):update_config(StorageOld, StorageNew); on_backend_update(BackendOld, BackendNew) when (BackendOld =:= undefined orelse is_tuple(BackendOld)) andalso (BackendNew =:= undefined orelse is_tuple(BackendNew)) -> - _ = emqx_maybe:apply(fun on_storage_stop/1, BackendOld), - _ = emqx_maybe:apply(fun on_storage_start/1, BackendNew), + _ = emqx_maybe:apply(fun stop_backend/1, BackendOld), + _ = emqx_maybe:apply(fun start_backend/1, BackendNew), ok. %%-------------------------------------------------------------------- @@ -185,10 +187,10 @@ on_backend_update(BackendOld, BackendNew) when backend(Config) -> emqx_ft_schema:backend(Config). -on_storage_start({Type, Storage}) -> +start_backend({Type, Storage}) -> (mod(Type)):start(Storage). -on_storage_stop({Type, Storage}) -> +stop_backend({Type, Storage}) -> (mod(Type)):stop(Storage). mod(local) -> diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl index 4c9cac67a..bc1b5fb4d 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter.erl @@ -31,7 +31,7 @@ -export([list/2]). %% Lifecycle API --export([on_config_update/2]). +-export([update_config/2]). %% Internal API -export([exporter/1]). @@ -81,7 +81,7 @@ -callback stop(exporter_conf()) -> ok. --callback update(exporter_conf(), exporter_conf()) -> +-callback update_config(exporter_conf(), exporter_conf()) -> ok | {error, _Reason}. %%------------------------------------------------------------------------------ @@ -148,8 +148,8 @@ list(Storage, Query) -> %% Lifecycle --spec on_config_update(storage(), storage()) -> ok | {error, term()}. -on_config_update(StorageOld, StorageNew) -> +-spec update_config(storage(), storage()) -> ok | {error, term()}. +update_config(StorageOld, StorageNew) -> on_exporter_update( emqx_maybe:apply(fun exporter/1, StorageOld), emqx_maybe:apply(fun exporter/1, StorageNew) @@ -158,7 +158,7 @@ on_config_update(StorageOld, StorageNew) -> on_exporter_update(Config, Config) -> ok; on_exporter_update({ExporterMod, ConfigOld}, {ExporterMod, ConfigNew}) -> - ExporterMod:update(ConfigOld, ConfigNew); + ExporterMod:update_config(ConfigOld, ConfigNew); on_exporter_update(ExporterOld, ExporterNew) -> _ = emqx_maybe:apply(fun stop/1, ExporterOld), _ = emqx_maybe:apply(fun start/1, ExporterNew), diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl index e37ba25af..9f2e5fd58 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_fs.erl @@ -31,7 +31,7 @@ -export([ start/1, stop/1, - update/2 + update_config/2 ]). %% Internal API for RPC @@ -161,8 +161,8 @@ start(_Options) -> ok. -spec stop(options()) -> ok. stop(_Options) -> ok. --spec update(options(), options()) -> ok. -update(_OldOptions, _NewOptions) -> ok. +-spec update_config(options(), options()) -> ok. +update_config(_OldOptions, _NewOptions) -> ok. %%-------------------------------------------------------------------- %% Internal API diff --git a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl index 4db2255f6..ac06ab957 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_exporter_s3.erl @@ -28,7 +28,12 @@ -export([ start/1, stop/1, - update/2 + update_config/2 +]). + +-export([ + pre_config_update/3, + post_config_update/3 ]). -type options() :: emqx_s3:profile_config(). @@ -112,12 +117,22 @@ start(Options) -> -spec stop(options()) -> ok. stop(_Options) -> - ok = emqx_s3:stop_profile(?S3_PROFILE_ID). + emqx_s3:stop_profile(?S3_PROFILE_ID). --spec update(options(), options()) -> ok. -update(_OldOptions, NewOptions) -> +-spec update_config(options(), options()) -> ok. +update_config(_OldOptions, NewOptions) -> emqx_s3:update_profile(?S3_PROFILE_ID, NewOptions). +%%-------------------------------------------------------------------- +%% Config update hooks +%%-------------------------------------------------------------------- + +pre_config_update(_ConfKey, NewOptions, OldOptions) -> + emqx_s3:pre_config_update(?S3_PROFILE_ID, NewOptions, OldOptions). + +post_config_update(_ConfKey, NewOptions, OldOptions) -> + emqx_s3:post_config_update(?S3_PROFILE_ID, NewOptions, OldOptions). + %%-------------------------------------------------------------------- %% Internal functions %% ------------------------------------------------------------------- diff --git a/apps/emqx_ft/src/emqx_ft_storage_fs.erl b/apps/emqx_ft/src/emqx_ft_storage_fs.erl index 85aa08405..1fd4d3a5d 100644 --- a/apps/emqx_ft/src/emqx_ft_storage_fs.erl +++ b/apps/emqx_ft/src/emqx_ft_storage_fs.erl @@ -48,9 +48,9 @@ -export([files/2]). --export([on_config_update/2]). -export([start/1]). -export([stop/1]). +-export([update_config/2]). -export_type([storage/0]). -export_type([filefrag/1]). @@ -230,10 +230,10 @@ files(Storage, Query) -> %% -on_config_update(StorageOld, StorageNew) -> +update_config(StorageOld, StorageNew) -> % NOTE: this will reset GC timer, frequent changes would postpone GC indefinitely ok = emqx_ft_storage_fs_gc:reset(StorageNew), - emqx_ft_storage_exporter:on_config_update(StorageOld, StorageNew). + emqx_ft_storage_exporter:update_config(StorageOld, StorageNew). start(Storage) -> ok = lists:foreach( @@ -242,11 +242,11 @@ start(Storage) -> end, child_spec(Storage) ), - ok = emqx_ft_storage_exporter:on_config_update(undefined, Storage), + ok = emqx_ft_storage_exporter:update_config(undefined, Storage), ok. stop(Storage) -> - ok = emqx_ft_storage_exporter:on_config_update(Storage, undefined), + ok = emqx_ft_storage_exporter:update_config(Storage, undefined), ok = lists:foreach( fun(#{id := ChildId}) -> _ = supervisor:terminate_child(emqx_ft_sup, ChildId), diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index bc0adf416..f61283eae 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -53,16 +53,13 @@ end_per_testcase(_Case, Config) -> t_update_config(_Config) -> ?assertMatch( {error, #{kind := validation_error}}, - emqx_conf:update( - [file_transfer], - #{<<"storage">> => #{<<"unknown">> => #{<<"foo">> => 42}}}, - #{} + emqx_ft_conf:update( + #{<<"storage">> => #{<<"unknown">> => #{<<"foo">> => 42}}} ) ), ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], + emqx_ft_conf:update( #{ <<"enable">> => true, <<"storage">> => #{ @@ -81,8 +78,7 @@ t_update_config(_Config) -> } } } - }, - #{} + } ) ), ?assertEqual( @@ -101,13 +97,8 @@ t_update_config(_Config) -> t_disable_restore_config(Config) -> ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], - #{ - <<"enable">> => true, - <<"storage">> => #{<<"local">> => #{}} - }, - #{} + emqx_ft_conf:update( + #{<<"enable">> => true, <<"storage">> => #{<<"local">> => #{}}} ) ), ?assertEqual( @@ -119,11 +110,7 @@ t_disable_restore_config(Config) -> % Verify that clearing storage settings reverts config to defaults ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], - #{<<"enable">> => false, <<"storage">> => undefined}, - #{} - ) + emqx_ft_conf:update(#{<<"enable">> => false, <<"storage">> => undefined}) ), ?assertEqual( false, @@ -155,8 +142,7 @@ t_disable_restore_config(Config) -> Root = emqx_ft_test_helpers:root(Config, node(), [segments]), ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], + emqx_ft_conf:update( #{ <<"enable">> => true, <<"storage">> => #{ @@ -167,8 +153,7 @@ t_disable_restore_config(Config) -> } } } - }, - #{} + } ) ), % Verify that GC is getting triggered eventually @@ -192,11 +177,7 @@ t_disable_restore_config(Config) -> t_switch_exporter(_Config) -> ?assertMatch( {ok, _}, - emqx_conf:update( - [file_transfer], - #{<<"enable">> => true}, - #{} - ) + emqx_ft_conf:update(#{<<"enable">> => true}) ), ?assertMatch( #{local := #{exporter := #{local := _}}}, @@ -248,5 +229,96 @@ t_switch_exporter(_Config) -> % Verify that transfers work ok = emqx_ft_test_helpers:upload_file(gen_clientid(), <<"f1">>, "f1", <>). +t_persist_ssl_certfiles(Config) -> + ?assertMatch( + {ok, _}, + emqx_ft_conf:update(mk_storage(true)) + ), + ?assertEqual( + [], + list_ssl_certfiles(Config) + ), + S3Config = #{ + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000 + }, + ?assertMatch( + {error, {pre_config_update, _, {bad_ssl_config, #{}}}}, + emqx_ft_conf:update( + mk_storage(true, #{ + <<"s3">> => S3Config#{ + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"certfile">> => <<"cert.pem">>, + <<"keyfile">> => <<"key.pem">> + } + } + } + }) + ) + ), + ?assertMatch( + {ok, _}, + emqx_ft_conf:update( + mk_storage(false, #{ + <<"s3">> => S3Config#{ + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), + <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() + } + } + } + }) + ) + ), + ?assertMatch( + #{ + local := #{ + exporter := #{ + s3 := #{ + transport_options := #{ + ssl := #{ + certfile := <<"/", _CertFilepath/binary>>, + keyfile := <<"/", _KeyFilepath/binary>> + } + } + } + } + } + }, + emqx_ft_conf:storage() + ), + ?assertMatch( + [_Certfile, _Keyfile], + list_ssl_certfiles(Config) + ), + ?assertMatch( + {ok, _}, + emqx_ft_conf:update(mk_storage(true)) + ), + ?assertEqual( + [], + list_ssl_certfiles(Config) + ). + +mk_storage(Enabled) -> + mk_storage(Enabled, #{<<"local">> => #{}}). + +mk_storage(Enabled, Exporter) -> + #{ + <<"enable">> => Enabled, + <<"storage">> => #{ + <<"local">> => #{ + <<"exporter">> => Exporter + } + } + }. + gen_clientid() -> emqx_base62:encode(emqx_guid:gen()). + +list_ssl_certfiles(_Config) -> + CertDir = emqx:mutable_certs_dir(), + filelib:fold_files(CertDir, ".*", true, fun(Filepath, Acc) -> [Filepath | Acc] end, []). diff --git a/apps/emqx_ft/test/emqx_ft_test_helpers.erl b/apps/emqx_ft/test/emqx_ft_test_helpers.erl index fbb3e7d6f..7a7d86d9a 100644 --- a/apps/emqx_ft/test/emqx_ft_test_helpers.erl +++ b/apps/emqx_ft/test/emqx_ft_test_helpers.erl @@ -136,3 +136,13 @@ upload_file(ClientId, FileId, Name, Data, Node) -> aws_config() -> emqx_s3_test_helpers:aws_config(tcp, binary_to_list(?S3_HOST), ?S3_PORT). + +pem_privkey() -> + << + "\n" + "-----BEGIN EC PRIVATE KEY-----\n" + "MHQCAQEEICKTbbathzvD8zvgjL7qRHhW4alS0+j0Loo7WeYX9AxaoAcGBSuBBAAK\n" + "oUQDQgAEJBdF7MIdam5T4YF3JkEyaPKdG64TVWCHwr/plC0QzNVJ67efXwxlVGTo\n" + "ju0VBj6tOX1y6C0U+85VOM0UU5xqvw==\n" + "-----END EC PRIVATE KEY-----\n" + >>. diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index cc48cdb93..3fa3c4b71 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -14,6 +14,11 @@ with_client/2 ]). +-export([ + pre_config_update/3, + post_config_update/3 +]). + -export_type([ profile_id/0, profile_config/0, @@ -94,3 +99,35 @@ with_client(ProfileId, Fun) when is_function(Fun, 1) andalso ?IS_PROFILE_ID(Prof {error, _} = Error -> Error end. + +%% + +-spec pre_config_update( + profile_id(), maybe(emqx_config:raw_config()), maybe(emqx_config:raw_config()) +) -> + {ok, maybe(profile_config())} | {error, term()}. +pre_config_update(ProfileId, NewConfig = #{<<"transport_options">> := TransportOpts}, _OldConfig) -> + case emqx_connector_ssl:convert_certs(mk_certs_dir(ProfileId), TransportOpts) of + {ok, TransportOptsConv} -> + {ok, NewConfig#{<<"transport_options">> := TransportOptsConv}}; + {error, Reason} -> + {error, Reason} + end; +pre_config_update(_ProfileId, NewConfig, _OldConfig) -> + {ok, NewConfig}. + +-spec post_config_update( + profile_id(), + maybe(emqx_config:config()), + maybe(emqx_config:config()) +) -> + ok. +post_config_update(ProfileId, NewConfig, OldConfig) -> + emqx_connector_ssl:try_clear_certs( + mk_certs_dir(ProfileId), + maps:get(transport_options, emqx_maybe:define(NewConfig, #{}), undefined), + maps:get(transport_options, emqx_maybe:define(OldConfig, #{}), undefined) + ). + +mk_certs_dir(ProfileId) -> + filename:join([s3, profiles, ProfileId]). From a2b03716beb3bf8650498746e32127a8e07453a4 Mon Sep 17 00:00:00 2001 From: Andrew Mayorov Date: Fri, 12 May 2023 14:47:38 +0300 Subject: [PATCH 57/92] feat(ft-api): provide configuration API To configure `emqx_ft` during the runtime. --- .../src/emqx_dashboard_swagger.erl | 16 +- .../test/emqx_swagger_parameter_SUITE.erl | 14 +- .../test/emqx_swagger_requestBody_SUITE.erl | 2 +- .../test/emqx_swagger_response_SUITE.erl | 6 +- apps/emqx_ft/src/emqx_ft_api.erl | 69 ++++++- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 171 +++++++++++++++++- apps/emqx_utils/src/emqx_utils.erl | 8 +- rel/i18n/emqx_ft_api.hocon | 6 + 8 files changed, 264 insertions(+), 28 deletions(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 9586d237d..94681d4c1 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -118,7 +118,7 @@ -type route_path() :: string() | binary(). -type route_methods() :: map(). -type route_handler() :: atom(). --type route_options() :: #{filter => filter() | undefined}. +-type route_options() :: #{filter => filter()}. -type api_spec_entry() :: {route_path(), route_methods(), route_handler(), route_options()}. -type api_spec_component() :: map(). @@ -137,10 +137,9 @@ spec(Module, Options) -> {ApiSpec, AllRefs} = lists:foldl( fun(Path, {AllAcc, AllRefsAcc}) -> - {OperationId, Specs, Refs} = parse_spec_ref(Module, Path, Options), - Opts = #{filter => filter(Options)}, + {OperationId, Specs, Refs, RouteOpts} = parse_spec_ref(Module, Path, Options), { - [{filename:join("/", Path), Specs, OperationId, Opts} | AllAcc], + [{filename:join("/", Path), Specs, OperationId, RouteOpts} | AllAcc], Refs ++ AllRefsAcc } end, @@ -350,6 +349,7 @@ parse_spec_ref(Module, Path, Options) -> ), error({failed_to_generate_swagger_spec, Module, Path}) end, + OperationId = maps:get('operationId', Schema), {Specs, Refs} = maps:fold( fun(Method, Meta, {Acc, RefsAcc}) -> (not lists:member(Method, ?METHODS)) andalso @@ -358,9 +358,13 @@ parse_spec_ref(Module, Path, Options) -> {Acc#{Method => Spec}, SubRefs ++ RefsAcc} end, {#{}, []}, - maps:without(['operationId'], Schema) + maps:without(['operationId', 'filter'], Schema) ), - {maps:get('operationId', Schema), Specs, Refs}. + RouteOpts = generate_route_opts(Schema, Options), + {OperationId, Specs, Refs, RouteOpts}. + +generate_route_opts(Schema, Options) -> + #{filter => compose_filters(filter(Options), custom_filter(Schema))}. check_parameters(Request, Spec, Module) -> #{bindings := Bindings, query_string := QueryStr} = Request, diff --git a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl index 81b3f4402..14d6f48b7 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_parameter_SUITE.erl @@ -108,8 +108,12 @@ t_ref(_Config) -> LocalPath = "/test/in/ref/local", Path = "/test/in/ref", Expect = [#{<<"$ref">> => <<"#/components/parameters/emqx_swagger_parameter_SUITE.page">>}], - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, LocalPath, #{}), + {OperationId, Spec, Refs, RouteOpts} = emqx_dashboard_swagger:parse_spec_ref( + ?MODULE, Path, #{} + ), + {OperationId, Spec, Refs, RouteOpts} = emqx_dashboard_swagger:parse_spec_ref( + ?MODULE, LocalPath, #{} + ), ?assertEqual(test, OperationId), Params = maps:get(parameters, maps:get(post, Spec)), ?assertEqual(Expect, Params), @@ -122,7 +126,7 @@ t_public_ref(_Config) -> #{<<"$ref">> => <<"#/components/parameters/public.page">>}, #{<<"$ref">> => <<"#/components/parameters/public.limit">>} ], - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Params = maps:get(parameters, maps:get(post, Spec)), ?assertEqual(Expect, Params), @@ -264,7 +268,7 @@ t_nullable(_Config) -> t_method(_Config) -> PathOk = "/method/ok", PathError = "/method/error", - {test, Spec, []} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, PathOk, #{}), + {test, Spec, [], #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, PathOk, #{}), ?assertEqual(lists:sort(?METHODS), lists:sort(maps:keys(Spec))), ?assertThrow( {error, #{module := ?MODULE, path := PathError, method := bar}}, @@ -393,7 +397,7 @@ assert_all_filters_equal(Spec, Filter) -> ). validate(Path, ExpectParams) -> - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Params = maps:get(parameters, maps:get(post, Spec)), ?assertEqual(ExpectParams, Params), diff --git a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl index 4bc0f4a7c..2457cd56a 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_requestBody_SUITE.erl @@ -719,7 +719,7 @@ t_object_trans_error(_Config) -> ok. validate(Path, ExpectSpec, ExpectRefs) -> - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), ?assertEqual(ExpectSpec, Spec), ?assertEqual(ExpectRefs, Refs), diff --git a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl index c0771f973..4488c7fc2 100644 --- a/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_swagger_response_SUITE.erl @@ -129,7 +129,7 @@ t_error(_Config) -> } } }, - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Response = maps:get(responses, maps:get(get, Spec)), ?assertEqual(Error400, maps:get(<<"400">>, Response)), @@ -375,7 +375,7 @@ t_complicated_type(_Config) -> } } }, - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Response = maps:get(responses, maps:get(post, Spec)), ?assertEqual(Object, maps:get(<<"200">>, Response)), @@ -665,7 +665,7 @@ schema("/fields/sub") -> to_schema(hoconsc:ref(sub_fields)). validate(Path, ExpectObject, ExpectRefs) -> - {OperationId, Spec, Refs} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), + {OperationId, Spec, Refs, #{}} = emqx_dashboard_swagger:parse_spec_ref(?MODULE, Path, #{}), ?assertEqual(test, OperationId), Response = maps:get(responses, maps:get(post, Spec)), ?assertEqual(ExpectObject, maps:get(<<"200">>, Response)), diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 7bc3a1d90..1ec0b6e31 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -40,27 +40,30 @@ %% API callbacks -export([ '/file_transfer/files'/2, - '/file_transfer/files/:clientid/:fileid'/2 + '/file_transfer/files/:clientid/:fileid'/2, + '/file_transfer'/2 ]). -import(hoconsc, [mk/2, ref/1, ref/2]). +-define(SCHEMA_CONFIG, ref(emqx_ft_schema, file_transfer)). + namespace() -> "file_transfer". api_spec() -> - emqx_dashboard_swagger:spec(?MODULE, #{ - check_schema => true, filter => fun ?MODULE:check_ft_enabled/2 - }). + emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}). paths() -> [ "/file_transfer/files", - "/file_transfer/files/:clientid/:fileid" + "/file_transfer/files/:clientid/:fileid", + "/file_transfer" ]. schema("/file_transfer/files") -> #{ 'operationId' => '/file_transfer/files', + filter => fun ?MODULE:check_ft_enabled/2, get => #{ tags => ?TAGS, summary => <<"List all uploaded files">>, @@ -83,6 +86,7 @@ schema("/file_transfer/files") -> schema("/file_transfer/files/:clientid/:fileid") -> #{ 'operationId' => '/file_transfer/files/:clientid/:fileid', + filter => fun ?MODULE:check_ft_enabled/2, get => #{ tags => ?TAGS, summary => <<"List files uploaded in a specific transfer">>, @@ -101,6 +105,36 @@ schema("/file_transfer/files/:clientid/:fileid") -> ) } } + }; +schema("/file_transfer") -> + #{ + 'operationId' => '/file_transfer', + get => #{ + tags => [<<"file_transfer">>], + summary => <<"Get current File Transfer configuration">>, + description => ?DESC("file_transfer_get_config"), + responses => #{ + 200 => ?SCHEMA_CONFIG, + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') + ) + } + }, + put => #{ + tags => [<<"file_transfer">>], + summary => <<"Update File Transfer configuration">>, + description => ?DESC("file_transfer_update_config"), + 'requestBody' => ?SCHEMA_CONFIG, + responses => #{ + 200 => ?SCHEMA_CONFIG, + 400 => emqx_dashboard_swagger:error_codes( + ['INVALID_CONFIG'], error_desc('INVALID_CONFIG') + ), + 503 => emqx_dashboard_swagger:error_codes( + ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') + ) + } + } }. check_ft_enabled(Params, _Meta) -> @@ -108,7 +142,7 @@ check_ft_enabled(Params, _Meta) -> true -> {ok, Params}; false -> - {503, error_msg('SERVICE_UNAVAILABLE', <<"Service unavailable">>)} + {503, error_msg('SERVICE_UNAVAILABLE')} end. '/file_transfer/files'(get, #{ @@ -147,6 +181,18 @@ check_ft_enabled(Params, _Meta) -> {503, error_msg('SERVICE_UNAVAILABLE')} end. +'/file_transfer'(get, _Meta) -> + {200, format_config(emqx_ft_conf:get())}; +'/file_transfer'(put, #{body := ConfigIn}) -> + case emqx_ft_conf:update(ConfigIn) of + {ok, #{config := Config}} -> + {200, format_config(Config)}; + {error, Error = #{kind := validation_error}} -> + {400, error_msg('INVALID_CONFIG', format_validation_error(Error))}; + {error, Error} -> + {400, error_msg('INVALID_CONFIG', emqx_utils:format(Error))} + end. + format_page(#{items := Files, cursor := Cursor}) -> #{ <<"files">> => lists:map(fun format_file_info/1, Files), @@ -157,14 +203,23 @@ format_page(#{items := Files}) -> <<"files">> => lists:map(fun format_file_info/1, Files) }. +format_config(Config) -> + Schema = emqx_hocon:make_schema(emqx_ft_schema:fields(file_transfer)), + hocon_tconf:make_serializable(Schema, emqx_utils_maps:binary_key_map(Config), #{}). + +format_validation_error(Error) -> + emqx_logger_jsonfmt:best_effort_json(Error). + error_msg(Code) -> #{code => Code, message => error_desc(Code)}. error_msg(Code, Msg) -> - #{code => Code, message => emqx_utils:readable_error_msg(Msg)}. + #{code => Code, message => Msg}. error_desc('FILES_NOT_FOUND') -> <<"Files requested for this transfer could not be found">>; +error_desc('INVALID_CONFIG') -> + <<"Provided configuration is invalid">>; error_desc('SERVICE_UNAVAILABLE') -> <<"Service unavailable">>. diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index 25ad42d75..ada1e49c3 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -108,6 +108,11 @@ init_per_testcase(Case, Config) -> [{tc, Case} | Config]. end_per_testcase(t_ft_disabled, _Config) -> emqx_config:put([file_transfer, enable], true); +end_per_testcase(t_configure, Config) -> + {ok, 200, _} = request(put, uri(["file_transfer"]), #{ + <<"enable">> => true, + <<"storage">> => emqx_ft_test_helpers:local_storage(Config) + }); end_per_testcase(_Case, _Config) -> ok. @@ -310,6 +315,155 @@ t_ft_disabled(Config) -> ) ). +t_configure(Config) -> + ?assertMatch( + {ok, 200, #{<<"enable">> := true, <<"storage">> := #{}}}, + request_json(get, uri(["file_transfer"]), Config) + ), + ?assertMatch( + {ok, 200, #{<<"enable">> := false}}, + request_json(put, uri(["file_transfer"]), #{<<"enable">> => false}, Config) + ), + ?assertMatch( + {ok, 200, #{<<"enable">> := false}}, + request_json(get, uri(["file_transfer"]), Config) + ), + ?assertMatch( + {ok, 200, #{}}, + request_json( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => emqx_ft_test_helpers:local_storage(Config) + }, + Config + ) + ), + ?assertMatch( + {ok, 400, _}, + request( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{}, + <<"remote">> => #{} + } + }, + Config + ) + ), + ?assertMatch( + {ok, 400, _}, + request( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"gc">> => #{<<"interval">> => -42} + } + } + }, + Config + ) + ), + S3Exporter = #{ + <<"host">> => <<"localhost">>, + <<"port">> => 9000, + <<"bucket">> => <<"emqx">>, + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"enable">> => true, + <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), + <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() + } + } + }, + ?assertMatch( + {ok, 200, #{ + <<"enable">> := true, + <<"storage">> := #{ + <<"local">> := #{ + <<"exporter">> := #{ + <<"s3">> := #{ + <<"transport_options">> := #{ + <<"ssl">> := #{ + <<"enable">> := true, + <<"certfile">> := <<"/", _CertFilepath/bytes>>, + <<"keyfile">> := <<"/", _KeyFilepath/bytes>> + } + } + } + } + } + } + }}, + request_json( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"exporter">> => #{ + <<"s3">> => S3Exporter + } + } + } + }, + Config + ) + ), + ?assertMatch( + {ok, 400, _}, + request_json( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"exporter">> => #{ + <<"s3">> => emqx_utils_maps:deep_put( + [<<"transport_options">>, <<"ssl">>, <<"keyfile">>], + S3Exporter, + <<>> + ) + } + } + } + }, + Config + ) + ), + ?assertMatch( + {ok, 200, #{}}, + request_json( + put, + uri(["file_transfer"]), + #{ + <<"enable">> => true, + <<"storage">> => #{ + <<"local">> => #{ + <<"exporter">> => #{ + <<"s3">> => emqx_utils_maps:deep_put( + [<<"transport_options">>, <<"ssl">>, <<"enable">>], + S3Exporter, + false + ) + } + } + } + }, + Config + ) + ), + ok. + %%-------------------------------------------------------------------- %% Helpers %%-------------------------------------------------------------------- @@ -332,17 +486,26 @@ mk_file_name(N) -> "file." ++ integer_to_list(N). request(Method, Url, Config) -> - Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, - emqx_mgmt_api_test_util:request_api(Method, Url, [], auth_header(Config), [], Opts). + request(Method, Url, [], Config). -request_json(Method, Url, Config) -> - case request(Method, Url, Config) of +request(Method, Url, Body, Config) -> + Opts = #{compatible_mode => true, httpc_req_opts => [{body_format, binary}]}, + request(Method, Url, Body, Opts, Config). + +request(Method, Url, Body, Opts, Config) -> + emqx_mgmt_api_test_util:request_api(Method, Url, Body, auth_header(Config), [], Opts). + +request_json(Method, Url, Body, Config) -> + case request(Method, Url, Body, [], Config) of {ok, Code, Body} -> {ok, Code, json(Body)}; Otherwise -> Otherwise end. +request_json(Method, Url, Config) -> + request_json(Method, Url, [], Config). + json(Body) when is_binary(Body) -> emqx_utils_json:decode(Body, [return_maps]). diff --git a/apps/emqx_utils/src/emqx_utils.erl b/apps/emqx_utils/src/emqx_utils.erl index 86667063c..80a9f8754 100644 --- a/apps/emqx_utils/src/emqx_utils.erl +++ b/apps/emqx_utils/src/emqx_utils.erl @@ -60,7 +60,8 @@ safe_filename/1, diff_lists/3, merge_lists/3, - tcp_keepalive_opts/4 + tcp_keepalive_opts/4, + format/1 ]). -export([ @@ -525,6 +526,9 @@ tcp_keepalive_opts({unix, darwin}, Idle, Interval, Probes) -> tcp_keepalive_opts(OS, _Idle, _Interval, _Probes) -> {error, {unsupported_os, OS}}. +format(Term) -> + iolist_to_binary(io_lib:format("~0p", [Term])). + %%------------------------------------------------------------------------------ %% Internal Functions %%------------------------------------------------------------------------------ @@ -606,7 +610,7 @@ to_hr_error({not_authorized, _}) -> to_hr_error({malformed_username_or_password, _}) -> <<"Bad username or password">>; to_hr_error(Error) -> - iolist_to_binary(io_lib:format("~0p", [Error])). + format(Error). try_to_existing_atom(Convert, Data, Encoding) -> try Convert(Data, Encoding) of diff --git a/rel/i18n/emqx_ft_api.hocon b/rel/i18n/emqx_ft_api.hocon index 9d88fcddd..81f908867 100644 --- a/rel/i18n/emqx_ft_api.hocon +++ b/rel/i18n/emqx_ft_api.hocon @@ -6,6 +6,12 @@ file_list.desc: file_list_transfer.desc: """List a file uploaded during specified transfer, identified by client id and file id.""" +file_transfer_get_config.desc: +"""Show current File Transfer configuration.""" + +file_transfer_update_config.desc: +"""Replace File Transfer configuration.""" + } emqx_ft_storage_exporter_fs_api { From a2eb658cd987f290f3072246db0eac20709def84 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 9 Jun 2023 10:34:07 +0300 Subject: [PATCH 58/92] feat(ft-api): do cleanup certs explicitly --- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 4 ---- apps/emqx_s3/src/emqx_s3.erl | 8 ++------ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index f61283eae..fc4391cde 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -297,10 +297,6 @@ t_persist_ssl_certfiles(Config) -> ?assertMatch( {ok, _}, emqx_ft_conf:update(mk_storage(true)) - ), - ?assertEqual( - [], - list_ssl_certfiles(Config) ). mk_storage(Enabled) -> diff --git a/apps/emqx_s3/src/emqx_s3.erl b/apps/emqx_s3/src/emqx_s3.erl index 3fa3c4b71..be91a19d2 100644 --- a/apps/emqx_s3/src/emqx_s3.erl +++ b/apps/emqx_s3/src/emqx_s3.erl @@ -122,12 +122,8 @@ pre_config_update(_ProfileId, NewConfig, _OldConfig) -> maybe(emqx_config:config()) ) -> ok. -post_config_update(ProfileId, NewConfig, OldConfig) -> - emqx_connector_ssl:try_clear_certs( - mk_certs_dir(ProfileId), - maps:get(transport_options, emqx_maybe:define(NewConfig, #{}), undefined), - maps:get(transport_options, emqx_maybe:define(OldConfig, #{}), undefined) - ). +post_config_update(_ProfileId, _NewConfig, _OldConfig) -> + ok. mk_certs_dir(ProfileId) -> filename:join([s3, profiles, ProfileId]). From fe691e8330084678c3f502f79c3f1a3e29d89d1b Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 9 Jun 2023 15:25:28 +0300 Subject: [PATCH 59/92] fix(ft-api): fix swagger schema dump for ft schema --- apps/emqx_dashboard/src/emqx_dashboard_swagger.erl | 2 ++ apps/emqx_s3/src/emqx_s3_schema.erl | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl index 94681d4c1..b0c78f0fe 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_swagger.erl @@ -902,6 +902,8 @@ typename_to_spec("json_binary()", _Mod) -> #{type => string, example => <<"{\"a\": [1,true]}">>}; typename_to_spec("port_number()", _Mod) -> range("1..65535"); +typename_to_spec("secret_access_key()", _Mod) -> + #{type => string, example => <<"TW8dPwmjpjJJuLW....">>}; typename_to_spec(Name, Mod) -> try_convert_to_spec(Name, Mod, [ fun try_remote_module_type/2, diff --git a/apps/emqx_s3/src/emqx_s3_schema.erl b/apps/emqx_s3/src/emqx_s3_schema.erl index c2460e20d..5fa57c230 100644 --- a/apps/emqx_s3/src/emqx_s3_schema.erl +++ b/apps/emqx_s3/src/emqx_s3_schema.erl @@ -14,6 +14,9 @@ -export([translate/1]). -export([translate/2]). +-type secret_access_key() :: string() | function(). +-reflect_type([secret_access_key/0]). + roots() -> [s3]. @@ -34,7 +37,7 @@ fields(s3) -> )}, {secret_access_key, mk( - hoconsc:union([string(), function()]), + secret_access_key(), #{ desc => ?DESC("secret_access_key"), required => false, From fde506838abd31c7223c2b664337457c760b185f Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Jul 2023 14:52:05 +0300 Subject: [PATCH 60/92] fix(ft-api): implement import config behaviour --- apps/emqx_ft/src/emqx_ft_conf.erl | 28 +++++++++++ apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 63 ++++++++++++++++++++---- apps/emqx_s3/src/emqx_s3.app.src | 2 +- 3 files changed, 83 insertions(+), 10 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft_conf.erl b/apps/emqx_ft/src/emqx_ft_conf.erl index 0907ffd09..f936b3056 100644 --- a/apps/emqx_ft/src/emqx_ft_conf.erl +++ b/apps/emqx_ft/src/emqx_ft_conf.erl @@ -19,6 +19,7 @@ -module(emqx_ft_conf). -behaviour(emqx_config_handler). +-behaviour(emqx_config_backup). -include_lib("emqx/include/logger.hrl"). @@ -45,6 +46,11 @@ post_config_update/5 ]). +%% callbacks for emqx_config_backup +-export([ + import_config/1 +]). + -type update_request() :: emqx_config:config(). -type milliseconds() :: non_neg_integer(). @@ -110,6 +116,24 @@ get() -> update(Config) -> emqx_conf:update([file_transfer], Config, #{override_to => cluster}). +%%---------------------------------------------------------------------------------------- +%% Data backup +%%---------------------------------------------------------------------------------------- + +import_config(#{<<"file_transfer">> := FTConf}) -> + OldFTConf = emqx:get_raw_config([file_transfer], #{}), + NewFTConf = maps:merge(OldFTConf, FTConf), + case emqx_conf:update([file_transfer], NewFTConf, #{override_to => cluster}) of + {ok, #{raw_config := NewRawConf}} -> + Changed = maps:get(changed, emqx_utils_maps:diff_maps(NewRawConf, FTConf)), + ChangedPaths = [[file_transfer, K] || K <- maps:keys(Changed)], + {ok, #{root_key => file_transfer, changed => ChangedPaths}}; + Error -> + {error, #{root_key => file_transfer, reason => Error}} + end; +import_config(_) -> + {ok, #{root_key => file_transfer, changed => []}}. + %%-------------------------------------------------------------------- %% emqx_config_handler callbacks %%-------------------------------------------------------------------- @@ -146,6 +170,10 @@ post_config_update([file_transfer | _], _Req, NewConfig, OldConfig, _AppEnvs) -> {error, Reason} end. +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + propagate_config_update(Fun, ConfKey, NewConfig, OldConfig) -> NewSubConf = emqx_utils_maps:deep_get(ConfKey, NewConfig, undefined), OldSubConf = emqx_utils_maps:deep_get(ConfKey, OldConfig, undefined), diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index fc4391cde..7316847aa 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -238,23 +238,18 @@ t_persist_ssl_certfiles(Config) -> [], list_ssl_certfiles(Config) ), - S3Config = #{ - <<"bucket">> => <<"emqx">>, - <<"host">> => <<"https://localhost">>, - <<"port">> => 9000 - }, ?assertMatch( {error, {pre_config_update, _, {bad_ssl_config, #{}}}}, emqx_ft_conf:update( mk_storage(true, #{ - <<"s3">> => S3Config#{ + <<"s3">> => mk_s3_config(#{ <<"transport_options">> => #{ <<"ssl">> => #{ <<"certfile">> => <<"cert.pem">>, <<"keyfile">> => <<"key.pem">> } } - } + }) }) ) ), @@ -262,14 +257,14 @@ t_persist_ssl_certfiles(Config) -> {ok, _}, emqx_ft_conf:update( mk_storage(false, #{ - <<"s3">> => S3Config#{ + <<"s3">> => mk_s3_config(#{ <<"transport_options">> => #{ <<"ssl">> => #{ <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() } } - } + }) }) ) ), @@ -299,6 +294,48 @@ t_persist_ssl_certfiles(Config) -> emqx_ft_conf:update(mk_storage(true)) ). +t_import(_Config) -> + {ok, _} = + emqx_ft_conf:update( + mk_storage(true, #{ + <<"s3">> => mk_s3_config(#{ + <<"transport_options">> => #{ + <<"ssl">> => #{ + <<"certfile">> => emqx_ft_test_helpers:pem_privkey(), + <<"keyfile">> => emqx_ft_test_helpers:pem_privkey() + } + } + }) + }) + ), + + {ok, #{filename := BackupFile}} = emqx_mgmt_data_backup:export(), + {ok, FileNames} = erl_tar:table(BackupFile, [compressed]), + [HoconFileName] = lists:filter( + fun(N) -> filename:basename(N) =:= "cluster.hocon" end, FileNames + ), + {ok, [{_, HoconConfig}]} = erl_tar:extract(BackupFile, [ + memory, compressed, {files, [HoconFileName]} + ]), + {ok, BackupConfig} = hocon:binary(HoconConfig), + FTBackupConfig = maps:with([<<"file_transfer">>], BackupConfig), + + {ok, _} = emqx_ft_conf:update(mk_storage(true)), + + ?assertMatch( + {ok, _}, + emqx_ft_conf:import_config(FTBackupConfig) + ), + + ?assertMatch( + #{local := #{exporter := #{s3 := #{enable := true}}}}, + emqx_ft_conf:storage() + ). + +%%-------------------------------------------------------------------- +%% Helper functions +%%-------------------------------------------------------------------- + mk_storage(Enabled) -> mk_storage(Enabled, #{<<"local">> => #{}}). @@ -312,6 +349,14 @@ mk_storage(Enabled, Exporter) -> } }. +mk_s3_config(S3Config) -> + BaseS3Config = #{ + <<"bucket">> => <<"emqx">>, + <<"host">> => <<"https://localhost">>, + <<"port">> => 9000 + }, + maps:merge(BaseS3Config, S3Config). + gen_clientid() -> emqx_base62:encode(emqx_guid:gen()). diff --git a/apps/emqx_s3/src/emqx_s3.app.src b/apps/emqx_s3/src/emqx_s3.app.src index 0599d7923..6dee7ed0a 100644 --- a/apps/emqx_s3/src/emqx_s3.app.src +++ b/apps/emqx_s3/src/emqx_s3.app.src @@ -1,6 +1,6 @@ {application, emqx_s3, [ {description, "EMQX S3"}, - {vsn, "5.0.8"}, + {vsn, "5.0.9"}, {modules, []}, {registered, [emqx_s3_sup]}, {applications, [ From e0353ab75093a43165852ecdd1f2e3d50cf5e6c5 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Tue, 4 Jul 2023 18:41:25 +0300 Subject: [PATCH 61/92] fix(ft-api): update FT API tests to use emqx_cth_suite --- apps/emqx/test/emqx_cth_suite.erl | 2 +- apps/emqx_ft/src/emqx_ft_api.erl | 8 +-- apps/emqx_ft/test/emqx_ft_api_SUITE.erl | 85 ++++++++----------------- 3 files changed, 29 insertions(+), 66 deletions(-) diff --git a/apps/emqx/test/emqx_cth_suite.erl b/apps/emqx/test/emqx_cth_suite.erl index aef0fc5e5..1ae6ceded 100644 --- a/apps/emqx/test/emqx_cth_suite.erl +++ b/apps/emqx/test/emqx_cth_suite.erl @@ -305,7 +305,7 @@ default_appspec(emqx_conf, SuiteOpts) -> #{ config => SharedConfig, % NOTE - % We inform `emqx` of our config loader before starting `emqx_conf` sothat it won't + % We inform `emqx` of our config loader before starting `emqx_conf` so that it won't % overwrite everything with a default configuration. before_start => fun() -> emqx_app:set_config_loader(?MODULE) diff --git a/apps/emqx_ft/src/emqx_ft_api.erl b/apps/emqx_ft/src/emqx_ft_api.erl index 1ec0b6e31..c4877fc68 100644 --- a/apps/emqx_ft/src/emqx_ft_api.erl +++ b/apps/emqx_ft/src/emqx_ft_api.erl @@ -114,10 +114,7 @@ schema("/file_transfer") -> summary => <<"Get current File Transfer configuration">>, description => ?DESC("file_transfer_get_config"), responses => #{ - 200 => ?SCHEMA_CONFIG, - 503 => emqx_dashboard_swagger:error_codes( - ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') - ) + 200 => ?SCHEMA_CONFIG } }, put => #{ @@ -129,9 +126,6 @@ schema("/file_transfer") -> 200 => ?SCHEMA_CONFIG, 400 => emqx_dashboard_swagger:error_codes( ['INVALID_CONFIG'], error_desc('INVALID_CONFIG') - ), - 503 => emqx_dashboard_swagger:error_codes( - ['SERVICE_UNAVAILABLE'], error_desc('SERVICE_UNAVAILABLE') ) } } diff --git a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl index ada1e49c3..ae8a5c01c 100644 --- a/apps/emqx_ft/test/emqx_ft_api_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_api_SUITE.erl @@ -24,58 +24,24 @@ -import(emqx_dashboard_api_test_helpers, [host/0, uri/1]). -all() -> - [ - {group, single}, - {group, cluster} - ]. - -groups() -> - [ - {single, [], emqx_common_test_helpers:all(?MODULE)}, - {cluster, [], emqx_common_test_helpers:all(?MODULE) -- [t_ft_disabled]} - ]. +all() -> emqx_common_test_helpers:all(?MODULE). suite() -> [{timetrap, {seconds, 90}}]. init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok. - -init_per_group(Group = single, Config) -> - WorkDir = ?config(priv_dir, Config), - Apps = emqx_cth_suite:start( - [ - {emqx, #{}}, - {emqx_ft, "file_transfer { enable = true }"}, - {emqx_management, #{}}, - {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"} - ], - #{work_dir => WorkDir} - ), - {ok, App} = emqx_common_test_http:create_default_app(), - [{group, Group}, {group_apps, Apps}, {api, App} | Config]; -init_per_group(Group = cluster, Config) -> WorkDir = ?config(priv_dir, Config), Cluster = mk_cluster_specs(Config), Nodes = [Node1 | _] = emqx_cth_cluster:start(Cluster, #{work_dir => WorkDir}), {ok, App} = erpc:call(Node1, emqx_common_test_http, create_default_app, []), - [{group, Group}, {cluster_nodes, Nodes}, {api, App} | Config]. + [{cluster_nodes, Nodes}, {api, App} | Config]. -end_per_group(single, Config) -> - {ok, _} = emqx_common_test_http:delete_default_app(), - ok = emqx_cth_suite:stop(?config(group_apps, Config)); -end_per_group(cluster, Config) -> - ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)); -end_per_group(_Group, _Config) -> - ok. +end_per_suite(Config) -> + ok = emqx_cth_cluster:stop(?config(cluster_nodes, Config)). mk_cluster_specs(_Config) -> Apps = [ - {emqx_conf, #{start => false}}, + emqx_conf, {emqx, #{override_env => [{boot_modules, [broker, listeners]}]}}, {emqx_ft, "file_transfer { enable = true }"}, {emqx_management, #{}} @@ -106,14 +72,8 @@ mk_cluster_specs(_Config) -> init_per_testcase(Case, Config) -> [{tc, Case} | Config]. -end_per_testcase(t_ft_disabled, _Config) -> - emqx_config:put([file_transfer, enable], true); -end_per_testcase(t_configure, Config) -> - {ok, 200, _} = request(put, uri(["file_transfer"]), #{ - <<"enable">> => true, - <<"storage">> => emqx_ft_test_helpers:local_storage(Config) - }); -end_per_testcase(_Case, _Config) -> +end_per_testcase(_Case, Config) -> + ok = reset_ft_config(Config, true), ok. %%-------------------------------------------------------------------- @@ -299,7 +259,7 @@ t_ft_disabled(Config) -> ) ), - ok = emqx_config:put([file_transfer, enable], false), + ok = reset_ft_config(Config, false), ?assertMatch( {ok, 503, _}, @@ -469,12 +429,7 @@ t_configure(Config) -> %%-------------------------------------------------------------------- test_nodes(Config) -> - case proplists:get_value(cluster_nodes, Config, []) of - [] -> - [node()]; - Nodes -> - Nodes - end. + ?config(cluster_nodes, Config). client_id(Config) -> iolist_to_binary(io_lib:format("~s.~s", [?config(group, Config), ?config(tc, Config)])). @@ -493,12 +448,12 @@ request(Method, Url, Body, Config) -> request(Method, Url, Body, Opts, Config). request(Method, Url, Body, Opts, Config) -> - emqx_mgmt_api_test_util:request_api(Method, Url, Body, auth_header(Config), [], Opts). + emqx_mgmt_api_test_util:request_api(Method, Url, [], auth_header(Config), Body, Opts). request_json(Method, Url, Body, Config) -> - case request(Method, Url, Body, [], Config) of - {ok, Code, Body} -> - {ok, Code, json(Body)}; + case request(Method, Url, Body, Config) of + {ok, Code, RespBody} -> + {ok, Code, json(RespBody)}; Otherwise -> Otherwise end. @@ -531,3 +486,17 @@ to_list(L) when is_list(L) -> pick(N, List) -> lists:nth(1 + (N rem length(List)), List). + +reset_ft_config(Config, Enable) -> + [Node | _] = test_nodes(Config), + LocalConfig = + #{ + <<"enable">> => Enable, + <<"storage">> => #{ + <<"local">> => #{ + <<"enable">> => true + } + } + }, + {ok, _} = rpc:call(Node, emqx_ft_conf, update, [LocalConfig]), + ok. From 56c81c2c258a35c81172d549cc1b813cf3f61cfe Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 7 Jul 2023 19:17:22 +0300 Subject: [PATCH 62/92] feat(ft-api): bump app versions --- apps/emqx_ft/src/emqx_ft.app.src | 2 +- apps/emqx_utils/src/emqx_utils.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_ft/src/emqx_ft.app.src b/apps/emqx_ft/src/emqx_ft.app.src index 8c37c77a8..ac498d6c6 100644 --- a/apps/emqx_ft/src/emqx_ft.app.src +++ b/apps/emqx_ft/src/emqx_ft.app.src @@ -1,6 +1,6 @@ {application, emqx_ft, [ {description, "EMQX file transfer over MQTT"}, - {vsn, "0.1.3"}, + {vsn, "0.1.4"}, {registered, []}, {mod, {emqx_ft_app, []}}, {applications, [ diff --git a/apps/emqx_utils/src/emqx_utils.app.src b/apps/emqx_utils/src/emqx_utils.app.src index df7d67321..5900514dc 100644 --- a/apps/emqx_utils/src/emqx_utils.app.src +++ b/apps/emqx_utils/src/emqx_utils.app.src @@ -2,7 +2,7 @@ {application, emqx_utils, [ {description, "Miscellaneous utilities for EMQX apps"}, % strict semver, bump manually! - {vsn, "5.0.4"}, + {vsn, "5.0.5"}, {modules, [ emqx_utils, emqx_utils_api, From 6337f52cf90fd7f8f3e51ac2af75024303273159 Mon Sep 17 00:00:00 2001 From: Thales Macedo Garitezi Date: Fri, 7 Jul 2023 13:33:34 -0300 Subject: [PATCH 63/92] fix(plugins): start/stop plugins when loading config from CLI Fixes https://emqx.atlassian.net/browse/EMQX-10288 --- apps/emqx_plugins/src/emqx_plugins.app.src | 2 +- apps/emqx_plugins/src/emqx_plugins.erl | 29 +++++++++ apps/emqx_plugins/src/emqx_plugins_app.erl | 4 ++ apps/emqx_plugins/test/emqx_plugins_SUITE.erl | 63 ++++++++++++++++++- changes/ce/fix-11229.en.md | 1 + 5 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 changes/ce/fix-11229.en.md diff --git a/apps/emqx_plugins/src/emqx_plugins.app.src b/apps/emqx_plugins/src/emqx_plugins.app.src index d5c16ea59..368a1ad46 100644 --- a/apps/emqx_plugins/src/emqx_plugins.app.src +++ b/apps/emqx_plugins/src/emqx_plugins.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_plugins, [ {description, "EMQX Plugin Management"}, - {vsn, "0.1.4"}, + {vsn, "0.1.5"}, {modules, []}, {mod, {emqx_plugins_app, []}}, {applications, [kernel, stdlib, emqx]}, diff --git a/apps/emqx_plugins/src/emqx_plugins.erl b/apps/emqx_plugins/src/emqx_plugins.erl index 04faa44e9..5181000de 100644 --- a/apps/emqx_plugins/src/emqx_plugins.erl +++ b/apps/emqx_plugins/src/emqx_plugins.erl @@ -51,6 +51,11 @@ get_tar/1 ]). +%% `emqx_config_handler' API +-export([ + post_config_update/5 +]). + %% internal -export([do_ensure_started/1]). -export([ @@ -857,3 +862,27 @@ running_apps() -> end, application:which_applications(infinity) ). + +%%-------------------------------------------------------------------- +%% `emqx_config_handler' API +%%-------------------------------------------------------------------- + +post_config_update([?CONF_ROOT], _Req, #{states := NewStates}, #{states := OldStates}, _Envs) -> + NewStatesIndex = maps:from_list([{NV, S} || S = #{name_vsn := NV} <- NewStates]), + OldStatesIndex = maps:from_list([{NV, S} || S = #{name_vsn := NV} <- OldStates]), + #{changed := Changed} = emqx_utils_maps:diff_maps(NewStatesIndex, OldStatesIndex), + maps:foreach(fun enable_disable_plugin/2, Changed), + ok; +post_config_update(_Path, _Req, _NewConf, _OldConf, _Envs) -> + ok. + +enable_disable_plugin(NameVsn, {#{enable := true}, #{enable := false}}) -> + %% errors are already logged in this fn + _ = ensure_stopped(NameVsn), + ok; +enable_disable_plugin(NameVsn, {#{enable := false}, #{enable := true}}) -> + %% errors are already logged in this fn + _ = ensure_started(NameVsn), + ok; +enable_disable_plugin(_NameVsn, _Diff) -> + ok. diff --git a/apps/emqx_plugins/src/emqx_plugins_app.erl b/apps/emqx_plugins/src/emqx_plugins_app.erl index c42936d56..f75089144 100644 --- a/apps/emqx_plugins/src/emqx_plugins_app.erl +++ b/apps/emqx_plugins/src/emqx_plugins_app.erl @@ -18,6 +18,8 @@ -behaviour(application). +-include("emqx_plugins.hrl"). + -export([ start/2, stop/1 @@ -27,7 +29,9 @@ start(_Type, _Args) -> %% load all pre-configured ok = emqx_plugins:ensure_started(), {ok, Sup} = emqx_plugins_sup:start_link(), + ok = emqx_config_handler:add_handler([?CONF_ROOT], emqx_plugins), {ok, Sup}. stop(_State) -> + ok = emqx_config_handler:remove_handler([?CONF_ROOT]), ok. diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index d6dee2c1e..9bb3f5e72 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -65,7 +65,7 @@ init_per_suite(Config) -> WorkDir = proplists:get_value(data_dir, Config), filelib:ensure_path(WorkDir), OrigInstallDir = emqx_plugins:get_config(install_dir, undefined), - emqx_common_test_helpers:start_apps([emqx_conf]), + emqx_common_test_helpers:start_apps([emqx_conf, emqx_plugins]), emqx_plugins:put_config(install_dir, WorkDir), [{orig_install_dir, OrigInstallDir} | Config]. @@ -77,7 +77,7 @@ end_per_suite(Config) -> undefined -> ok; OrigInstallDir -> emqx_plugins:put_config(install_dir, OrigInstallDir) end, - emqx_common_test_helpers:stop_apps([emqx_conf]), + emqx_common_test_helpers:stop_apps([emqx_plugins, emqx_conf]), ok. init_per_testcase(TestCase, Config) -> @@ -505,6 +505,65 @@ t_elixir_plugin(Config) -> ?assertEqual([], emqx_plugins:list()), ok. +t_load_config_from_cli({init, Config}) -> + #{package := Package} = get_demo_plugin_package(), + NameVsn = filename:basename(Package, ?PACKAGE_SUFFIX), + [{name_vsn, NameVsn} | Config]; +t_load_config_from_cli({'end', Config}) -> + NameVsn = ?config(name_vsn, Config), + ok = emqx_plugins:ensure_stopped(NameVsn), + ok = emqx_plugins:ensure_uninstalled(NameVsn), + ok; +t_load_config_from_cli(Config) when is_list(Config) -> + NameVsn = ?config(name_vsn, Config), + ok = emqx_plugins:ensure_installed(NameVsn), + ?assertEqual([], emqx_plugins:configured()), + ok = emqx_plugins:ensure_enabled(NameVsn), + ok = emqx_plugins:ensure_started(NameVsn), + Params0 = unused, + ?assertMatch( + {200, [#{running_status := [#{status := running}]}]}, + emqx_mgmt_api_plugins:list_plugins(get, Params0) + ), + + %% Now we disable it via CLI loading + Conf0 = emqx_config:get([plugins]), + ?assertMatch( + #{states := [#{enable := true}]}, + Conf0 + ), + #{states := [Plugin0]} = Conf0, + Conf1 = Conf0#{states := [Plugin0#{enable := false}]}, + Filename = filename:join(["/tmp", [?FUNCTION_NAME, ".hocon"]]), + ok = file:write_file(Filename, hocon_pp:do(#{plugins => Conf1}, #{})), + ok = emqx_conf_cli:conf(["load", Filename]), + + Conf2 = emqx_config:get([plugins]), + ?assertMatch( + #{states := [#{enable := false}]}, + Conf2 + ), + ?assertMatch( + {200, [#{running_status := [#{status := stopped}]}]}, + emqx_mgmt_api_plugins:list_plugins(get, Params0) + ), + + %% Re-enable it via CLI loading + ok = file:write_file(Filename, hocon_pp:do(#{plugins => Conf0}, #{})), + ok = emqx_conf_cli:conf(["load", Filename]), + + Conf3 = emqx_config:get([plugins]), + ?assertMatch( + #{states := [#{enable := true}]}, + Conf3 + ), + ?assertMatch( + {200, [#{running_status := [#{status := running}]}]}, + emqx_mgmt_api_plugins:list_plugins(get, Params0) + ), + + ok. + group_t_copy_plugin_to_a_new_node({init, Config}) -> WorkDir = proplists:get_value(data_dir, Config), FromInstallDir = filename:join(WorkDir, atom_to_list(plugins_copy_from)), diff --git a/changes/ce/fix-11229.en.md b/changes/ce/fix-11229.en.md new file mode 100644 index 000000000..864f545fe --- /dev/null +++ b/changes/ce/fix-11229.en.md @@ -0,0 +1 @@ +Fixed an issue preventing plugins from starting/stopping after changing configuration via `emqx ctl conf load`. From b568bda09821db218ca82be527fec8aecd2b228c Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 7 Jul 2023 19:39:05 +0300 Subject: [PATCH 64/92] feat(ft-api): simplify config import test --- apps/emqx_ft/test/emqx_ft_conf_SUITE.erl | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl index 7316847aa..3fdfdf65a 100644 --- a/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl +++ b/apps/emqx_ft/test/emqx_ft_conf_SUITE.erl @@ -309,15 +309,7 @@ t_import(_Config) -> }) ), - {ok, #{filename := BackupFile}} = emqx_mgmt_data_backup:export(), - {ok, FileNames} = erl_tar:table(BackupFile, [compressed]), - [HoconFileName] = lists:filter( - fun(N) -> filename:basename(N) =:= "cluster.hocon" end, FileNames - ), - {ok, [{_, HoconConfig}]} = erl_tar:extract(BackupFile, [ - memory, compressed, {files, [HoconFileName]} - ]), - {ok, BackupConfig} = hocon:binary(HoconConfig), + BackupConfig = emqx_config:get_raw([]), FTBackupConfig = maps:with([<<"file_transfer">>], BackupConfig), {ok, _} = emqx_ft_conf:update(mk_storage(true)), From 7de26a17765fffdac1e6d3d37507b58f1974c48d Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Thu, 1 Jun 2023 23:19:19 +0300 Subject: [PATCH 65/92] feat(authz): use extensible map format for actions in authz rules * support authorization on retain, qos fields * refactored authz tests heavily --- apps/emqx/include/emqx_access_control.hrl | 14 + apps/emqx/include/emqx_placeholder.hrl | 4 +- apps/emqx/src/emqx_access_control.erl | 28 +- apps/emqx/src/emqx_authz_cache.erl | 5 +- apps/emqx/src/emqx_channel.erl | 47 +- apps/emqx/src/emqx_types.erl | 8 +- apps/emqx/test/emqx_access_control_SUITE.erl | 17 +- apps/emqx/test/emqx_authz_cache_SUITE.erl | 2 - apps/emqx/test/emqx_channel_SUITE.erl | 3 +- apps/emqx/test/emqx_proper_types.erl | 21 +- apps/emqx_authz/include/emqx_authz.hrl | 62 +- apps/emqx_authz/src/emqx_authz.erl | 27 + apps/emqx_authz/src/emqx_authz_api_mnesia.erl | 101 +-- apps/emqx_authz/src/emqx_authz_file.erl | 1 - apps/emqx_authz/src/emqx_authz_http.erl | 35 +- apps/emqx_authz/src/emqx_authz_mnesia.erl | 28 +- apps/emqx_authz/src/emqx_authz_mongodb.erl | 26 +- apps/emqx_authz/src/emqx_authz_mysql.erl | 52 +- apps/emqx_authz/src/emqx_authz_postgresql.erl | 51 +- apps/emqx_authz/src/emqx_authz_redis.erl | 58 +- apps/emqx_authz/src/emqx_authz_rule.erl | 183 +++-- apps/emqx_authz/src/emqx_authz_rule_raw.erl | 197 +++++ apps/emqx_authz/src/emqx_authz_utils.erl | 30 +- .../test/emqx_authz_api_mnesia_SUITE.erl | 6 +- .../emqx_authz/test/emqx_authz_file_SUITE.erl | 65 +- .../emqx_authz/test/emqx_authz_http_SUITE.erl | 93 ++- apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl | 5 +- .../test/emqx_authz_mnesia_SUITE.erl | 189 +++-- .../test/emqx_authz_mongodb_SUITE.erl | 516 +++++++------ .../test/emqx_authz_mysql_SUITE.erl | 564 +++++++------- .../test/emqx_authz_postgresql_SUITE.erl | 582 ++++++++------- .../test/emqx_authz_redis_SUITE.erl | 363 +++++---- .../emqx_authz/test/emqx_authz_rule_SUITE.erl | 696 ++++++++++++++---- .../test/emqx_authz_rule_raw_SUITE.erl | 274 +++++++ apps/emqx_authz/test/emqx_authz_test_lib.erl | 266 ++----- apps/emqx_exhook/src/emqx_exhook.app.src | 2 +- apps/emqx_exhook/src/emqx_exhook_handler.erl | 11 +- apps/emqx_exhook/test/emqx_exhook_SUITE.erl | 8 +- .../test/props/prop_exhook_hooks.erl | 10 +- apps/emqx_gateway/src/emqx_gateway_ctx.erl | 4 +- .../src/emqx_coap_channel.erl | 7 +- .../src/emqx_coap_pubsub_handler.erl | 75 +- .../src/emqx_exproto_channel.erl | 7 +- .../src/emqx_lwm2m_channel.erl | 16 +- .../src/emqx_gateway_mqttsn.app.src | 2 +- .../src/emqx_mqttsn_channel.erl | 11 +- .../src/emqx_stomp_channel.erl | 10 +- .../test/emqx_mgmt_api_clients_SUITE.erl | 35 +- .../emqx_rule_engine/src/emqx_rule_events.erl | 6 +- changes/ee/feat-11132.en.md | 2 + rel/i18n/emqx_authz_api_mnesia.hocon | 12 +- 51 files changed, 3144 insertions(+), 1693 deletions(-) create mode 100644 apps/emqx_authz/src/emqx_authz_rule_raw.erl create mode 100644 apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl create mode 100644 changes/ee/feat-11132.en.md diff --git a/apps/emqx/include/emqx_access_control.hrl b/apps/emqx/include/emqx_access_control.hrl index 693bc91b5..e840d2b4a 100644 --- a/apps/emqx/include/emqx_access_control.hrl +++ b/apps/emqx/include/emqx_access_control.hrl @@ -18,3 +18,17 @@ -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, "authorization"). -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM, authorization). -define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY, <<"authorization">>). + +-define(DEFAULT_ACTION_QOS, 0). +-define(DEFAULT_ACTION_RETAIN, false). + +-define(AUTHZ_SUBSCRIBE(QOS), #{action_type => subscribe, qos => QOS}). +-define(AUTHZ_SUBSCRIBE, ?AUTHZ_SUBSCRIBE(?DEFAULT_ACTION_QOS)). + +-define(AUTHZ_PUBLISH(QOS, RETAIN), #{action_type => publish, qos => QOS, retain => RETAIN}). +-define(AUTHZ_PUBLISH(QOS), ?AUTHZ_PUBLISH(QOS, ?DEFAULT_ACTION_RETAIN)). +-define(AUTHZ_PUBLISH, ?AUTHZ_PUBLISH(?DEFAULT_ACTION_QOS)). + +-define(authz_action(PUBSUB, QOS), #{action_type := PUBSUB, qos := QOS}). +-define(authz_action(PUBSUB), ?authz_action(PUBSUB, _)). +-define(authz_action, ?authz_action(_)). diff --git a/apps/emqx/include/emqx_placeholder.hrl b/apps/emqx/include/emqx_placeholder.hrl index d5da3fb18..7b2ce6c6b 100644 --- a/apps/emqx/include/emqx_placeholder.hrl +++ b/apps/emqx/include/emqx_placeholder.hrl @@ -21,7 +21,7 @@ -define(PH(Type), <<"${", Type/binary, "}">>). -%% action: publish/subscribe/all +%% action: publish/subscribe -define(PH_ACTION, <<"${action}">>). %% cert @@ -79,6 +79,7 @@ -define(PH_REASON, <<"${reason}">>). -define(PH_ENDPOINT_NAME, <<"${endpoint_name}">>). +-define(PH_RETAIN, <<"${retain}">>). %% sync change these place holder with binary def. -define(PH_S_ACTION, "${action}"). @@ -113,5 +114,6 @@ -define(PH_S_NODE, "${node}"). -define(PH_S_REASON, "${reason}"). -define(PH_S_ENDPOINT_NAME, "${endpoint_name}"). +-define(PH_S_RETAIN, "${retain}"). -endif. diff --git a/apps/emqx/src/emqx_access_control.erl b/apps/emqx/src/emqx_access_control.erl index efe9bee37..43669bf6c 100644 --- a/apps/emqx/src/emqx_access_control.erl +++ b/apps/emqx/src/emqx_access_control.erl @@ -77,10 +77,10 @@ authenticate(Credential) -> %% @doc Check Authorization -spec authorize(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic()) -> allow | deny. -authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) -> +authorize(ClientInfo, Action, <<"$delayed/", Data/binary>> = RawTopic) -> case binary:split(Data, <<"/">>) of [_, Topic] -> - authorize(ClientInfo, PubSub, Topic); + authorize(ClientInfo, Action, Topic); _ -> ?SLOG(warning, #{ msg => "invalid_delayed_topic_format", @@ -90,39 +90,39 @@ authorize(ClientInfo, PubSub, <<"$delayed/", Data/binary>> = RawTopic) -> inc_authz_metrics(deny), deny end; -authorize(ClientInfo, PubSub, Topic) -> +authorize(ClientInfo, Action, Topic) -> Result = case emqx_authz_cache:is_enabled() of - true -> check_authorization_cache(ClientInfo, PubSub, Topic); - false -> do_authorize(ClientInfo, PubSub, Topic) + true -> check_authorization_cache(ClientInfo, Action, Topic); + false -> do_authorize(ClientInfo, Action, Topic) end, inc_authz_metrics(Result), Result. -check_authorization_cache(ClientInfo, PubSub, Topic) -> - case emqx_authz_cache:get_authz_cache(PubSub, Topic) of +check_authorization_cache(ClientInfo, Action, Topic) -> + case emqx_authz_cache:get_authz_cache(Action, Topic) of not_found -> - AuthzResult = do_authorize(ClientInfo, PubSub, Topic), - emqx_authz_cache:put_authz_cache(PubSub, Topic, AuthzResult), + AuthzResult = do_authorize(ClientInfo, Action, Topic), + emqx_authz_cache:put_authz_cache(Action, Topic, AuthzResult), AuthzResult; AuthzResult -> emqx:run_hook( 'client.check_authz_complete', - [ClientInfo, PubSub, Topic, AuthzResult, cache] + [ClientInfo, Action, Topic, AuthzResult, cache] ), inc_authz_metrics(cache_hit), AuthzResult end. -do_authorize(ClientInfo, PubSub, Topic) -> +do_authorize(ClientInfo, Action, Topic) -> NoMatch = emqx:get_config([authorization, no_match], allow), Default = #{result => NoMatch, from => default}, - case run_hooks('client.authorize', [ClientInfo, PubSub, Topic], Default) of + case run_hooks('client.authorize', [ClientInfo, Action, Topic], Default) of AuthzResult = #{result := Result} when Result == allow; Result == deny -> From = maps:get(from, AuthzResult, unknown), emqx:run_hook( 'client.check_authz_complete', - [ClientInfo, PubSub, Topic, Result, From] + [ClientInfo, Action, Topic, Result, From] ), Result; Other -> @@ -133,7 +133,7 @@ do_authorize(ClientInfo, PubSub, Topic) -> }), emqx:run_hook( 'client.check_authz_complete', - [ClientInfo, PubSub, Topic, deny, unknown_return_format] + [ClientInfo, Action, Topic, deny, unknown_return_format] ), deny end. diff --git a/apps/emqx/src/emqx_authz_cache.erl b/apps/emqx/src/emqx_authz_cache.erl index 6555266a5..af19ecf8f 100644 --- a/apps/emqx/src/emqx_authz_cache.erl +++ b/apps/emqx/src/emqx_authz_cache.erl @@ -16,7 +16,7 @@ -module(emqx_authz_cache). --include("emqx.hrl"). +-include("emqx_access_control.hrl"). -export([ list_authz_cache/0, @@ -159,8 +159,7 @@ dump_authz_cache() -> map_authz_cache(Fun) -> [ Fun(R) - || R = {{SubPub, _T}, _Authz} <- erlang:get(), - SubPub =:= publish orelse SubPub =:= subscribe + || R = {{?authz_action, _T}, _Authz} <- erlang:get() ]. foreach_authz_cache(Fun) -> _ = map_authz_cache(Fun), diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 93bb6535e..01af1a7b5 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -20,6 +20,7 @@ -include("emqx.hrl"). -include("emqx_channel.hrl"). -include("emqx_mqtt.hrl"). +-include("emqx_access_control.hrl"). -include("logger.hrl"). -include("types.hrl"). @@ -491,7 +492,7 @@ handle_in( ok -> TopicFilters0 = parse_topic_filters(TopicFilters), TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0), - TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel), + TupleTopicFilters0 = check_sub_authzs(SubPkt, TopicFilters1, Channel), HasAuthzDeny = lists:any( fun({_TopicFilter, ReasonCode}) -> ReasonCode =:= ?RC_NOT_AUTHORIZED @@ -1838,14 +1839,34 @@ check_pub_alias( check_pub_alias(_Packet, _Channel) -> ok. +%%-------------------------------------------------------------------- +%% Athorization action + +authz_action(#mqtt_packet{ + header = #mqtt_packet_header{qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{} +}) -> + ?AUTHZ_PUBLISH(QoS, Retain); +authz_action(#mqtt_packet{ + header = #mqtt_packet_header{qos = QoS}, variable = #mqtt_packet_subscribe{} +}) -> + ?AUTHZ_SUBSCRIBE(QoS); +%% Will message +authz_action(#message{qos = QoS, flags = #{retain := Retain}}) -> + ?AUTHZ_PUBLISH(QoS, Retain); +authz_action(#message{qos = QoS}) -> + ?AUTHZ_PUBLISH(QoS). + %%-------------------------------------------------------------------- %% Check Pub Authorization check_pub_authz( - #mqtt_packet{variable = #mqtt_packet_publish{topic_name = Topic}}, + #mqtt_packet{ + variable = #mqtt_packet_publish{topic_name = Topic} + } = Packet, #channel{clientinfo = ClientInfo} ) -> - case emqx_access_control:authorize(ClientInfo, publish, Topic) of + Action = authz_action(Packet), + case emqx_access_control:authorize(ClientInfo, Action, Topic) of allow -> ok; deny -> {error, ?RC_NOT_AUTHORIZED} end. @@ -1868,24 +1889,23 @@ check_pub_caps( %%-------------------------------------------------------------------- %% Check Sub Authorization -%% TODO: not only check topic filter. Qos chould be checked too. -%% Not implemented yet: -%% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7] -check_sub_authzs(TopicFilters, Channel) -> - check_sub_authzs(TopicFilters, Channel, []). +check_sub_authzs(Packet, TopicFilters, Channel) -> + Action = authz_action(Packet), + check_sub_authzs(Action, TopicFilters, Channel, []). check_sub_authzs( + Action, [TopicFilter = {Topic, _} | More], Channel = #channel{clientinfo = ClientInfo}, Acc ) -> - case emqx_access_control:authorize(ClientInfo, subscribe, Topic) of + case emqx_access_control:authorize(ClientInfo, Action, Topic) of allow -> - check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]); + check_sub_authzs(Action, More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]); deny -> - check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) + check_sub_authzs(Action, More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc]) end; -check_sub_authzs([], _Channel, Acc) -> +check_sub_authzs(_Action, [], _Channel, Acc) -> lists:reverse(Acc). %%-------------------------------------------------------------------- @@ -2149,7 +2169,8 @@ publish_will_msg( ClientInfo = #{mountpoint := MountPoint}, Msg = #message{topic = Topic} ) -> - PublishingDisallowed = emqx_access_control:authorize(ClientInfo, publish, Topic) =/= allow, + Action = authz_action(Msg), + PublishingDisallowed = emqx_access_control:authorize(ClientInfo, Action, Topic) =/= allow, ClientBanned = emqx_banned:check(ClientInfo), case PublishingDisallowed orelse ClientBanned of true -> diff --git a/apps/emqx/src/emqx_types.erl b/apps/emqx/src/emqx_types.erl index c6a567318..cc937f81c 100644 --- a/apps/emqx/src/emqx_types.erl +++ b/apps/emqx/src/emqx_types.erl @@ -29,6 +29,7 @@ -export_type([ zone/0, pubsub/0, + pubsub_action/0, subid/0 ]). @@ -127,7 +128,12 @@ | exactly_once. -type zone() :: atom(). --type pubsub() :: publish | subscribe. +-type pubsub_action() :: publish | subscribe. + +-type pubsub() :: + #{action_type := subscribe, qos := qos()} + | #{action_type := publish, qos := qos(), retain := boolean()}. + -type subid() :: binary() | atom(). -type group() :: binary() | undefined. diff --git a/apps/emqx/test/emqx_access_control_SUITE.erl b/apps/emqx/test/emqx_access_control_SUITE.erl index 305eaf5eb..5d4344de6 100644 --- a/apps/emqx/test/emqx_access_control_SUITE.erl +++ b/apps/emqx/test/emqx_access_control_SUITE.erl @@ -19,8 +19,8 @@ -compile(export_all). -compile(nowarn_export_all). --include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("eunit/include/eunit.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). @@ -44,8 +44,7 @@ t_authenticate(_) -> ?assertMatch({ok, _}, emqx_access_control:authenticate(clientinfo())). t_authorize(_) -> - Publish = ?PUBLISH_PACKET(?QOS_0, <<"t">>, 1, <<"payload">>), - ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish, <<"t">>)). + ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, <<"t">>)). t_delayed_authorize(_) -> RawTopic = <<"$delayed/1/foo/2">>, @@ -54,11 +53,11 @@ t_delayed_authorize(_) -> ok = emqx_hooks:put('client.authorize', {?MODULE, authz_stub, [Topic]}, ?HP_AUTHZ), - Publish1 = ?PUBLISH_PACKET(?QOS_0, RawTopic, 1, <<"payload">>), - ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), Publish1, RawTopic)), + ?assertEqual(allow, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, RawTopic)), - Publish2 = ?PUBLISH_PACKET(?QOS_0, InvalidTopic, 1, <<"payload">>), - ?assertEqual(deny, emqx_access_control:authorize(clientinfo(), Publish2, InvalidTopic)), + ?assertEqual( + deny, emqx_access_control:authorize(clientinfo(), ?AUTHZ_PUBLISH, InvalidTopic) + ), ok. t_quick_deny_anonymous(_) -> @@ -96,8 +95,8 @@ t_quick_deny_anonymous(_) -> %% Helper functions %%-------------------------------------------------------------------- -authz_stub(_Client, _PubSub, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}}; -authz_stub(_Client, _PubSub, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}. +authz_stub(_Client, _Action, ValidTopic, _DefaultResult, ValidTopic) -> {stop, #{result => allow}}; +authz_stub(_Client, _Action, _Topic, _DefaultResult, _ValidTopic) -> {stop, #{result => deny}}. quick_deny_anonymous_authn(#{username := <<"badname">>}, _AuthResult) -> {stop, {error, not_authorized}}; diff --git a/apps/emqx/test/emqx_authz_cache_SUITE.erl b/apps/emqx/test/emqx_authz_cache_SUITE.erl index 5497422af..09d1e1522 100644 --- a/apps/emqx/test/emqx_authz_cache_SUITE.erl +++ b/apps/emqx/test/emqx_authz_cache_SUITE.erl @@ -43,8 +43,6 @@ t_clean_authz_cache(_) -> ct:sleep(100), ClientPid = case emqx_cm:lookup_channels(<<"emqx_c">>) of - [Pid] when is_pid(Pid) -> - Pid; Pids when is_list(Pids) -> lists:last(Pids); _ -> diff --git a/apps/emqx/test/emqx_channel_SUITE.erl b/apps/emqx/test/emqx_channel_SUITE.erl index 5653cd2d2..f266dbcfa 100644 --- a/apps/emqx/test/emqx_channel_SUITE.erl +++ b/apps/emqx/test/emqx_channel_SUITE.erl @@ -908,7 +908,8 @@ t_check_pub_alias(_) -> t_check_sub_authzs(_) -> emqx_config:put_zone_conf(default, [authorization, enable], true), TopicFilter = {<<"t">>, ?DEFAULT_SUBOPTS}, - [{TopicFilter, 0}] = emqx_channel:check_sub_authzs([TopicFilter], channel()). + Subscribe = ?SUBSCRIBE_PACKET(1, [TopicFilter]), + [{TopicFilter, 0}] = emqx_channel:check_sub_authzs(Subscribe, [TopicFilter], channel()). t_enrich_connack_caps(_) -> ok = meck:new(emqx_mqtt_caps, [passthrough, no_history]), diff --git a/apps/emqx/test/emqx_proper_types.erl b/apps/emqx/test/emqx_proper_types.erl index 95cf29bee..6d1ced486 100644 --- a/apps/emqx/test/emqx_proper_types.erl +++ b/apps/emqx/test/emqx_proper_types.erl @@ -20,6 +20,7 @@ -include_lib("proper/include/proper.hrl"). -include("emqx.hrl"). +-include("emqx_access_control.hrl"). %% High level Types -export([ @@ -34,7 +35,8 @@ subopts/0, nodename/0, normal_topic/0, - normal_topic_filter/0 + normal_topic_filter/0, + pubsub/0 ]). %% Basic Types @@ -482,6 +484,23 @@ normal_topic_filter() -> end ). +subscribe_action() -> + ?LET( + Qos, + qos(), + ?AUTHZ_SUBSCRIBE(Qos) + ). + +publish_action() -> + ?LET( + {Qos, Retain}, + {qos(), boolean()}, + ?AUTHZ_PUBLISH(Qos, Retain) + ). + +pubsub() -> + oneof([publish_action(), subscribe_action()]). + %%-------------------------------------------------------------------- %% Basic Types %%-------------------------------------------------------------------- diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index b43a2cdab..8676da134 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -18,16 +18,6 @@ -define(APP, emqx_authz). --define(ALLOW_DENY(A), - ((A =:= allow) orelse (A =:= <<"allow">>) orelse - (A =:= deny) orelse (A =:= <<"deny">>)) -). --define(PUBSUB(A), - ((A =:= subscribe) orelse (A =:= <<"subscribe">>) orelse - (A =:= publish) orelse (A =:= <<"publish">>) orelse - (A =:= all) orelse (A =:= <<"all">>)) -). - %% authz_mnesia -define(ACL_TABLE, emqx_acl). @@ -72,6 +62,20 @@ topic => <<"eq test/#">>, permission => <<"deny">>, action => <<"all">> + }, + #{ + topic => <<"test/toopic/3">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"1">>], + retain => <<"true">> + }, + #{ + topic => <<"test/toopic/4">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"0">>, <<"1">>, <<"2">>], + retain => <<"all">> } ] }). @@ -92,6 +96,20 @@ topic => <<"eq test/#">>, permission => <<"deny">>, action => <<"all">> + }, + #{ + topic => <<"test/toopic/3">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"1">>], + retain => <<"true">> + }, + #{ + topic => <<"test/toopic/4">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"0">>, <<"1">>, <<"2">>], + retain => <<"all">> } ] }). @@ -111,9 +129,28 @@ topic => <<"eq test/#">>, permission => <<"deny">>, action => <<"all">> + }, + #{ + topic => <<"test/toopic/3">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"1">>], + retain => <<"true">> + }, + #{ + topic => <<"test/toopic/4">>, + permission => <<"allow">>, + action => <<"publish">>, + qos => [<<"0">>, <<"1">>, <<"2">>], + retain => <<"all">> } ] }). + +-define(USERNAME_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?USERNAME_RULES_EXAMPLE))). +-define(CLIENTID_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?CLIENTID_RULES_EXAMPLE))). +-define(ALL_RULES_EXAMPLE_COUNT, length(maps:get(rules, ?ALL_RULES_EXAMPLE))). + -define(META_EXAMPLE, #{ page => 1, limit => 100, @@ -121,3 +158,8 @@ }). -define(RESOURCE_GROUP, <<"emqx_authz">>). + +-define(AUTHZ_FEATURES, [rich_actions]). + +-define(DEFAULT_RULE_QOS, [0, 1, 2]). +-define(DEFAULT_RULE_RETAIN, all). diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 6f45a88b7..b5f1a1298 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -39,6 +39,11 @@ get_enabled_authzs/0 ]). +-export([ + feature_available/1, + set_feature_available/2 +]). + -export([post_config_update/5, pre_config_update/3]). -export([acl_conf_file/0]). @@ -519,6 +524,28 @@ read_acl_file(#{<<"path">> := Path} = Source) -> {ok, Rules} = emqx_authz_file:read_file(Path), maps:remove(<<"path">>, Source#{<<"rules">> => Rules}). +%%------------------------------------------------------------------------------ +%% Extednded Features +%%------------------------------------------------------------------------------ + +-if(?EMQX_RELEASE_EDITION == ee). + +-define(DEFAULT_RICH_ACTIONS, true). + +-else. + +-define(DEFAULT_RICH_ACTIONS, false). + +-endif. + +-define(FEATURE_KEY(_NAME_), {?MODULE, _NAME_}). + +feature_available(rich_actions) -> + persistent_term:get(?FEATURE_KEY(rich_actions), ?DEFAULT_RICH_ACTIONS). + +set_feature_available(Feature, Enable) when is_boolean(Enable) -> + persistent_term:put(?FEATURE_KEY(Feature), Enable). + %%------------------------------------------------------------------------------ %% Internal function %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl index f5ac40f5e..a2a8f2525 100644 --- a/apps/emqx_authz/src/emqx_authz_api_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_api_mnesia.erl @@ -359,6 +359,22 @@ fields(rule_item) -> required => true, example => publish } + )}, + {qos, + mk( + array(emqx_schema:qos()), + #{ + desc => ?DESC(qos), + default => ?DEFAULT_RULE_QOS + } + )}, + {retain, + mk( + hoconsc:union([all, boolean()]), + #{ + desc => ?DESC(retain), + default => ?DEFAULT_RULE_RETAIN + } )} ]; fields(clientid) -> @@ -434,7 +450,7 @@ users(post, #{body := Body}) when is_list(Body) -> [] -> lists:foreach( fun(#{<<"username">> := Username, <<"rules">> := Rules}) -> - emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)) + emqx_authz_mnesia:store_rules({username, Username}, Rules) end, Body ), @@ -470,7 +486,7 @@ clients(post, #{body := Body}) when is_list(Body) -> [] -> lists:foreach( fun(#{<<"clientid">> := ClientID, <<"rules">> := Rules}) -> - emqx_authz_mnesia:store_rules({clientid, ClientID}, format_rules(Rules)) + emqx_authz_mnesia:store_rules({clientid, ClientID}, Rules) end, Body ), @@ -489,21 +505,14 @@ user(get, #{bindings := #{username := Username}}) -> {ok, Rules} -> {200, #{ username => Username, - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }} end; user(put, #{ bindings := #{username := Username}, body := #{<<"username">> := Username, <<"rules">> := Rules} }) -> - emqx_authz_mnesia:store_rules({username, Username}, format_rules(Rules)), + emqx_authz_mnesia:store_rules({username, Username}, Rules), {204}; user(delete, #{bindings := #{username := Username}}) -> case emqx_authz_mnesia:get_rules({username, Username}) of @@ -521,21 +530,14 @@ client(get, #{bindings := #{clientid := ClientID}}) -> {ok, Rules} -> {200, #{ clientid => ClientID, - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }} end; client(put, #{ bindings := #{clientid := ClientID}, body := #{<<"clientid">> := ClientID, <<"rules">> := Rules} }) -> - emqx_authz_mnesia:store_rules({clientid, ClientID}, format_rules(Rules)), + emqx_authz_mnesia:store_rules({clientid, ClientID}, Rules), {204}; client(delete, #{bindings := #{clientid := ClientID}}) -> case emqx_authz_mnesia:get_rules({clientid, ClientID}) of @@ -552,18 +554,11 @@ all(get, _) -> {200, #{rules => []}}; {ok, Rules} -> {200, #{ - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }} end; all(post, #{body := #{<<"rules">> := Rules}}) -> - emqx_authz_mnesia:store_rules(all, format_rules(Rules)), + emqx_authz_mnesia:store_rules(all, Rules), {204}; all(delete, _) -> emqx_authz_mnesia:store_rules(all, []), @@ -626,58 +621,20 @@ run_fuzzy_filter( %%-------------------------------------------------------------------- %% format funcs -%% format rule from api -format_rules(Rules) when is_list(Rules) -> - lists:foldl( - fun( - #{ - <<"topic">> := Topic, - <<"action">> := Action, - <<"permission">> := Permission - }, - AccIn - ) when - ?PUBSUB(Action) andalso - ?ALLOW_DENY(Permission) - -> - AccIn ++ [{atom(Permission), atom(Action), Topic}] - end, - [], - Rules - ). - %% format result from mnesia tab format_result([{username, Username}, {rules, Rules}]) -> #{ username => Username, - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }; format_result([{clientid, ClientID}, {rules, Rules}]) -> #{ clientid => ClientID, - rules => [ - #{ - topic => Topic, - action => Action, - permission => Permission - } - || {Permission, Action, Topic} <- Rules - ] + rules => format_rules(Rules) }. -atom(B) when is_binary(B) -> - try - binary_to_existing_atom(B, utf8) - catch - _Error:_Expection -> binary_to_atom(B) - end; -atom(A) when is_atom(A) -> A. + +format_rules(Rules) -> + [emqx_authz_rule_raw:format_rule(Rule) || Rule <- Rules]. %%-------------------------------------------------------------------- %% Internal functions diff --git a/apps/emqx_authz/src/emqx_authz_file.erl b/apps/emqx_authz/src/emqx_authz_file.erl index 317395a45..7d421d39b 100644 --- a/apps/emqx_authz/src/emqx_authz_file.erl +++ b/apps/emqx_authz/src/emqx_authz_file.erl @@ -16,7 +16,6 @@ -module(emqx_authz_file). --include("emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). -behaviour(emqx_authz). diff --git a/apps/emqx_authz/src/emqx_authz_http.erl b/apps/emqx_authz/src/emqx_authz_http.erl index 5747e6eeb..aafbe25ad 100644 --- a/apps/emqx_authz/src/emqx_authz_http.erl +++ b/apps/emqx_authz/src/emqx_authz_http.erl @@ -51,6 +51,11 @@ ?PH_CERT_CN_NAME ]). +-define(PLACEHOLDERS_FOR_RICH_ACTIONS, [ + ?PH_QOS, + ?PH_RETAIN +]). + description() -> "AuthZ with http". @@ -72,7 +77,7 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ type := http, @@ -81,7 +86,7 @@ authorize( request_timeout := RequestTimeout } = Config ) -> - Request = generate_request(PubSub, Topic, Client, Config), + Request = generate_request(Action, Topic, Client, Config), case emqx_resource:simple_sync_query(ResourceID, {Method, Request, RequestTimeout}) of {ok, 204, _Headers} -> {matched, allow}; @@ -139,14 +144,14 @@ parse_config( method => Method, base_url => BaseUrl, headers => Headers, - base_path_templete => emqx_authz_utils:parse_str(Path, ?PLACEHOLDERS), + base_path_templete => emqx_authz_utils:parse_str(Path, placeholders()), base_query_template => emqx_authz_utils:parse_deep( cow_qs:parse_qs(to_bin(Query)), - ?PLACEHOLDERS + placeholders() ), body_template => emqx_authz_utils:parse_deep( maps:to_list(maps:get(body, Conf, #{})), - ?PLACEHOLDERS + placeholders() ), request_timeout => ReqTimeout, %% pool_type default value `random` @@ -173,7 +178,7 @@ parse_url(Url) -> end. generate_request( - PubSub, + Action, Topic, Client, #{ @@ -184,7 +189,7 @@ generate_request( body_template := BodyTemplate } ) -> - Values = client_vars(Client, PubSub, Topic), + Values = client_vars(Client, Action, Topic), Path = emqx_authz_utils:render_urlencoded_str(BasePathTemplate, Values), Query = emqx_authz_utils:render_deep(BaseQueryTemplate, Values), Body = emqx_authz_utils:render_deep(BodyTemplate, Values), @@ -227,11 +232,9 @@ serialize_body(<<"application/json">>, Body) -> serialize_body(<<"application/x-www-form-urlencoded">>, Body) -> query_string(Body). -client_vars(Client, PubSub, Topic) -> - Client#{ - action => PubSub, - topic => Topic - }. +client_vars(Client, Action, Topic) -> + Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), + Vars#{topic => Topic}. to_list(A) when is_atom(A) -> atom_to_list(A); @@ -243,3 +246,11 @@ to_list(L) when is_list(L) -> to_bin(B) when is_binary(B) -> B; to_bin(L) when is_list(L) -> list_to_binary(L); to_bin(X) -> X. + +placeholders() -> + placeholders(emqx_authz:feature_available(rich_actions)). + +placeholders(true) -> + ?PLACEHOLDERS ++ ?PLACEHOLDERS_FOR_RICH_ACTIONS; +placeholders(false) -> + ?PLACEHOLDERS. diff --git a/apps/emqx_authz/src/emqx_authz_mnesia.erl b/apps/emqx_authz/src/emqx_authz_mnesia.erl index bdb4877c0..2cecd0c71 100644 --- a/apps/emqx_authz/src/emqx_authz_mnesia.erl +++ b/apps/emqx_authz/src/emqx_authz_mnesia.erl @@ -16,7 +16,6 @@ -module(emqx_authz_mnesia). --include_lib("emqx/include/emqx.hrl"). -include_lib("stdlib/include/ms_transform.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -202,25 +201,16 @@ record_count() -> %%-------------------------------------------------------------------- normalize_rules(Rules) -> - lists:map(fun normalize_rule/1, Rules). + lists:flatmap(fun normalize_rule/1, Rules). -normalize_rule({Permission, Action, Topic}) -> - {normalize_permission(Permission), normalize_action(Action), normalize_topic(Topic)}; -normalize_rule(Rule) -> - error({invalid_rule, Rule}). - -normalize_topic(Topic) when is_list(Topic) -> list_to_binary(Topic); -normalize_topic(Topic) when is_binary(Topic) -> Topic; -normalize_topic(Topic) -> error({invalid_rule_topic, Topic}). - -normalize_action(publish) -> publish; -normalize_action(subscribe) -> subscribe; -normalize_action(all) -> all; -normalize_action(Action) -> error({invalid_rule_action, Action}). - -normalize_permission(allow) -> allow; -normalize_permission(deny) -> deny; -normalize_permission(Permission) -> error({invalid_rule_permission, Permission}). +normalize_rule(RuleRaw) -> + case emqx_authz_rule_raw:parse_rule(RuleRaw) of + %% For backward compatibility + {ok, {Permission, Action, TopicFilters}} -> + [{Permission, Action, TopicFilter} || TopicFilter <- TopicFilters]; + {error, Reason} -> + error(Reason) + end. do_get_rules(Key) -> case mnesia:dirty_read(?ACL_TABLE, Key) of diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index e82ff64e1..7adb6d2d9 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -68,7 +68,7 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ collection := Collection, @@ -97,15 +97,21 @@ authorize( {ok, []} -> nomatch; {ok, Rows} -> - Rules = [ - emqx_authz_rule:compile({Permission, all, Action, Topics}) - || #{ - <<"topics">> := Topics, - <<"permission">> := Permission, - <<"action">> := Action - } <- Rows - ], - do_authorize(Client, PubSub, Topic, Rules) + Rules = lists:flatmap(fun parse_rule/1, Rows), + do_authorize(Client, Action, Topic, Rules) + end. + +parse_rule(Row) -> + case emqx_authz_rule_raw:parse_rule(Row) of + {ok, {Permission, Action, Topics}} -> + [emqx_authz_rule:compile({Permission, all, Action, Topics})]; + {error, Reason} -> + ?SLOG(error, #{ + msg => "parse_rule_error", + reason => Reason, + row => Row + }), + [] end. do_authorize(_Client, _PubSub, _Topic, []) -> diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index fb6a29c3d..01debaea9 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -55,7 +55,7 @@ create(#{query := SQL} = Source0) -> ResourceId = emqx_authz_utils:make_resource_id(?MODULE), Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}}, {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source), - Source#{annotations => #{id => ResourceId, tmpl_oken => TmplToken}}. + Source#{annotations => #{id => ResourceId, tmpl_token => TmplToken}}. update(#{query := SQL} = Source0) -> {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS), @@ -64,7 +64,7 @@ update(#{query := SQL} = Source0) -> {error, Reason} -> error({load_config_error, Reason}); {ok, Id} -> - Source#{annotations => #{id => Id, tmpl_oken => TmplToken}} + Source#{annotations => #{id => Id, tmpl_token => TmplToken}} end. destroy(#{annotations := #{id := Id}}) -> @@ -72,57 +72,51 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ annotations := #{ id := ResourceID, - tmpl_oken := TmplToken + tmpl_token := TmplToken } } ) -> - RenderParams = emqx_authz_utils:render_sql_params(TmplToken, Client), + Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), + RenderParams = emqx_authz_utils:render_sql_params(TmplToken, Vars), case emqx_resource:simple_sync_query(ResourceID, {prepared_query, ?PREPARE_KEY, RenderParams}) of - {ok, _Columns, []} -> + {ok, _ColumnNames, []} -> nomatch; - {ok, Columns, Rows} -> - do_authorize(Client, PubSub, Topic, Columns, Rows); + {ok, ColumnNames, Rows} -> + do_authorize(Client, Action, Topic, ColumnNames, Rows); {error, Reason} -> ?SLOG(error, #{ msg => "query_mysql_error", reason => Reason, - tmpl_oken => TmplToken, + tmpl_token => TmplToken, params => RenderParams, resource_id => ResourceID }), nomatch end. -do_authorize(_Client, _PubSub, _Topic, _Columns, []) -> +do_authorize(_Client, _Action, _Topic, _ColumnNames, []) -> nomatch; -do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) -> - case +do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) -> + try emqx_authz_rule:match( - Client, - PubSub, - Topic, - emqx_authz_rule:compile(format_result(Columns, Row)) + Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row) ) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail) + nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail) + catch + error:Reason -> + ?SLOG(error, #{ + msg => "match_rule_error", + reason => Reason, + rule => Row + }), + do_authorize(Client, Action, Topic, ColumnNames, Tail) end. - -format_result(Columns, Row) -> - Permission = lists:nth(index(<<"permission">>, Columns), Row), - Action = lists:nth(index(<<"action">>, Columns), Row), - Topic = lists:nth(index(<<"topic">>, Columns), Row), - {Permission, all, Action, [Topic]}. - -index(Elem, List) -> - index(Elem, List, 1). -index(_Elem, [], _Index) -> {error, not_found}; -index(Elem, [Elem | _List], Index) -> Index; -index(Elem, [_ | List], Index) -> index(Elem, List, Index + 1). diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index 05f2315a6..1b05451cc 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -21,6 +21,8 @@ -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include_lib("epgsql/include/epgsql.hrl"). + -behaviour(emqx_authz). %% AuthZ Callbacks @@ -77,7 +79,7 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ annotations := #{ @@ -86,14 +88,15 @@ authorize( } } ) -> - RenderedParams = emqx_authz_utils:render_sql_params(Placeholders, Client), + Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), + RenderedParams = emqx_authz_utils:render_sql_params(Placeholders, Vars), case emqx_resource:simple_sync_query(ResourceID, {prepared_query, ResourceID, RenderedParams}) of {ok, _Columns, []} -> nomatch; {ok, Columns, Rows} -> - do_authorize(Client, PubSub, Topic, Columns, Rows); + do_authorize(Client, Action, Topic, column_names(Columns), Rows); {error, Reason} -> ?SLOG(error, #{ msg => "query_postgresql_error", @@ -104,33 +107,29 @@ authorize( nomatch end. -do_authorize(_Client, _PubSub, _Topic, _Columns, []) -> +do_authorize(_Client, _Action, _Topic, _ColumnNames, []) -> nomatch; -do_authorize(Client, PubSub, Topic, Columns, [Row | Tail]) -> - case +do_authorize(Client, Action, Topic, ColumnNames, [Row | Tail]) -> + try emqx_authz_rule:match( - Client, - PubSub, - Topic, - emqx_authz_rule:compile(format_result(Columns, Row)) + Client, Action, Topic, emqx_authz_utils:parse_rule_from_row(ColumnNames, Row) ) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, PubSub, Topic, Columns, Tail) + nomatch -> do_authorize(Client, Action, Topic, ColumnNames, Tail) + catch + error:Reason:Stack -> + ?SLOG(error, #{ + msg => "match_rule_error", + reason => Reason, + rule => Row, + stack => Stack + }), + do_authorize(Client, Action, Topic, ColumnNames, Tail) end. -format_result(Columns, Row) -> - Permission = lists:nth(index(<<"permission">>, 2, Columns), erlang:tuple_to_list(Row)), - Action = lists:nth(index(<<"action">>, 2, Columns), erlang:tuple_to_list(Row)), - Topic = lists:nth(index(<<"topic">>, 2, Columns), erlang:tuple_to_list(Row)), - {Permission, all, Action, [Topic]}. - -index(Key, N, TupleList) when is_integer(N) -> - Tuple = lists:keyfind(Key, N, TupleList), - index(Tuple, TupleList, 1); -index(_Tuple, [], _Index) -> - {error, not_found}; -index(Tuple, [Tuple | _TupleList], Index) -> - Index; -index(Tuple, [_ | TupleList], Index) -> - index(Tuple, TupleList, Index + 1). +column_names(Columns) -> + lists:map( + fun(#column{name = Name}) -> Name end, + Columns + ). diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 3b19db832..01149b5bb 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -70,19 +70,20 @@ destroy(#{annotations := #{id := Id}}) -> authorize( Client, - PubSub, + Action, Topic, #{ cmd_template := CmdTemplate, annotations := #{id := ResourceID} } ) -> - Cmd = emqx_authz_utils:render_deep(CmdTemplate, Client), + Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), + Cmd = emqx_authz_utils:render_deep(CmdTemplate, Vars), case emqx_resource:simple_sync_query(ResourceID, {cmd, Cmd}) of {ok, []} -> nomatch; {ok, Rows} -> - do_authorize(Client, PubSub, Topic, Rows); + do_authorize(Client, Action, Topic, Rows); {error, Reason} -> ?SLOG(error, #{ msg => "query_redis_error", @@ -93,21 +94,60 @@ authorize( nomatch end. -do_authorize(_Client, _PubSub, _Topic, []) -> +do_authorize(_Client, _Action, _Topic, []) -> nomatch; -do_authorize(Client, PubSub, Topic, [TopicFilter, Action | Tail]) -> - case +do_authorize(Client, Action, Topic, [TopicFilterRaw, RuleEncoded | Tail]) -> + try emqx_authz_rule:match( Client, - PubSub, + Action, Topic, - emqx_authz_rule:compile({allow, all, Action, [TopicFilter]}) + compile_rule(RuleEncoded, TopicFilterRaw) ) of {matched, Permission} -> {matched, Permission}; - nomatch -> do_authorize(Client, PubSub, Topic, Tail) + nomatch -> do_authorize(Client, Action, Topic, Tail) + catch + error:Reason -> + ?SLOG(error, #{ + msg => "match_rule_error", + reason => Reason, + rule_encoded => RuleEncoded, + topic_filter_raw => TopicFilterRaw + }), + do_authorize(Client, Action, Topic, Tail) + end. + +compile_rule(RuleBin, TopicFilterRaw) -> + RuleRaw = + maps:merge( + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => TopicFilterRaw + }, + parse_rule(RuleBin) + ), + case emqx_authz_rule_raw:parse_rule(RuleRaw) of + {ok, {Permission, Action, Topics}} -> + emqx_authz_rule:compile({Permission, all, Action, Topics}); + {error, Reason} -> + error(Reason) end. tokens(Query) -> Tokens = binary:split(Query, <<" ">>, [global]), [Token || Token <- Tokens, size(Token) > 0]. + +parse_rule(<<"publish">>) -> + #{<<"action">> => <<"publish">>}; +parse_rule(<<"subscribe">>) -> + #{<<"action">> => <<"subscribe">>}; +parse_rule(<<"all">>) -> + #{<<"action">> => <<"all">>}; +parse_rule(Bin) when is_binary(Bin) -> + case emqx_utils_json:safe_decode(Bin, [return_maps]) of + {ok, Map} when is_map(Map) -> + maps:with([<<"qos">>, <<"action">>, <<"retain">>], Map); + {error, Error} -> + error({invalid_topic_rule, Bin, Error}) + end. diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index ec1a8c5de..a59679a8d 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -16,9 +16,9 @@ -module(emqx_authz_rule). --include("emqx_authz.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). +-include("emqx_authz.hrl"). -ifdef(TEST). -compile(export_all). @@ -32,50 +32,123 @@ compile/1 ]). --type ipaddress() :: - {ipaddr, esockd_cidr:cidr_string()} - | {ipaddrs, list(esockd_cidr:cidr_string())}. +-type permission() :: allow | deny. --type username() :: {username, binary()}. - --type clientid() :: {clientid, binary()}. - --type who() :: +-type who_condition() :: ipaddress() | username() | clientid() | {'and', [ipaddress() | username() | clientid()]} | {'or', [ipaddress() | username() | clientid()]} | all. +-type ipaddress() :: + {ipaddr, esockd_cidr:cidr_string()} + | {ipaddrs, list(esockd_cidr:cidr_string())}. +-type username() :: {username, binary()}. +-type clientid() :: {clientid, binary()}. --type action() :: subscribe | publish | all. --type permission() :: allow | deny. +-type action_condition() :: + subscribe + | publish + | #{action_type := subscribe, qos := qos_condition()} + | #{action_type := publish | all, qos := qos_condition(), retain := retain_condition()} + | all. +-type qos_condition() :: [qos()]. +-type retain_condition() :: retain() | all. --type rule() :: {permission(), who(), action(), list(emqx_types:topic())}. +-type topic_condition() :: list(emqx_types:topic() | {eq, emqx_types:topic()}). + +-type rule() :: {permission(), who_condition(), action_condition(), topic_condition()}. + +-type qos() :: 0..2. +-type retain() :: boolean(). +-type action() :: + #{action_type := subscribe, qos := qos()} + | #{action_type := publish, qos := qos(), retain := retain()}. -export_type([ - action/0, - permission/0 + permission/0, + who_condition/0, + action_condition/0, + topic_condition/0 ]). +-define(IS_PERMISSION(Permission), (Permission =:= allow orelse Permission =:= deny)). + compile({Permission, all}) when - ?ALLOW_DENY(Permission) + ?IS_PERMISSION(Permission) -> {Permission, all, all, [compile_topic(<<"#">>)]}; compile({Permission, Who, Action, TopicFilters}) when - ?ALLOW_DENY(Permission), ?PUBSUB(Action), is_list(TopicFilters) + ?IS_PERMISSION(Permission) andalso is_list(TopicFilters) -> - {atom(Permission), compile_who(Who), atom(Action), [ + {Permission, compile_who(Who), compile_action(Action), [ compile_topic(Topic) || Topic <- TopicFilters ]}; -compile({Permission, _Who, _Action, _TopicFilter}) when not ?ALLOW_DENY(Permission) -> +compile({Permission, _Who, _Action, _TopicFilter}) when not ?IS_PERMISSION(Permission) -> throw({invalid_authorization_permission, Permission}); -compile({_Permission, _Who, Action, _TopicFilter}) when not ?PUBSUB(Action) -> - throw({invalid_authorization_action, Action}); compile(BadRule) -> throw({invalid_authorization_rule, BadRule}). +compile_action(Action) -> + compile_action(emqx_authz:feature_available(rich_actions), Action). + +-define(IS_ACTION_WITH_RETAIN(Action), (Action =:= publish orelse Action =:= all)). + +compile_action(_RichActionsOn, subscribe) -> + subscribe; +compile_action(_RichActionsOn, Action) when ?IS_ACTION_WITH_RETAIN(Action) -> + Action; +compile_action(true = _RichActionsOn, {subscribe, Opts}) when is_list(Opts) -> + #{ + action_type => subscribe, + qos => qos_from_opts(Opts) + }; +compile_action(true = _RichActionsOn, {Action, Opts}) when + ?IS_ACTION_WITH_RETAIN(Action) andalso is_list(Opts) +-> + #{ + action_type => Action, + qos => qos_from_opts(Opts), + retain => retain_from_opts(Opts) + }; +compile_action(_RichActionsOn, Action) -> + throw({invalid_authorization_action, Action}). + +qos_from_opts(Opts) -> + try + case proplists:get_all_values(qos, Opts) of + [] -> + ?DEFAULT_RULE_QOS; + QoSs -> + lists:flatmap( + fun + (QoS) when is_integer(QoS) -> + [validate_qos(QoS)]; + (QoS) when is_list(QoS) -> + lists:map(fun validate_qos/1, QoS) + end, + QoSs + ) + end + catch + bad_qos -> + throw({invalid_authorization_qos, Opts}) + end. + +validate_qos(QoS) when is_integer(QoS), QoS >= 0, QoS =< 2 -> + QoS; +validate_qos(_) -> + throw(bad_qos). + +retain_from_opts(Opts) -> + case proplists:get_value(retain, Opts, ?DEFAULT_RULE_RETAIN) of + all -> all; + Retain when is_boolean(Retain) -> Retain; + _ -> throw({invalid_authorization_retain, Opts}) + end. + compile_who(all) -> all; compile_who({user, Username}) -> @@ -99,8 +172,12 @@ compile_who({ipaddrs, CIDRs}) -> compile_who({'and', L}) when is_list(L) -> {'and', [compile_who(Who) || Who <- L]}; compile_who({'or', L}) when is_list(L) -> - {'or', [compile_who(Who) || Who <- L]}. + {'or', [compile_who(Who) || Who <- L]}; +compile_who(Who) -> + throw({invalid_who, Who}). +compile_topic("eq " ++ Topic) -> + {eq, emqx_topic:words(bin(Topic))}; compile_topic(<<"eq ", Topic/binary>>) -> {eq, emqx_topic:words(Topic)}; compile_topic({eq, Topic}) -> @@ -117,45 +194,65 @@ compile_topic(Topic) -> Tokens -> {pattern, Tokens} end. -atom(B) when is_binary(B) -> - try - binary_to_existing_atom(B, utf8) - catch - _E:_S -> binary_to_atom(B) - end; -atom(A) when is_atom(A) -> A. - bin(L) when is_list(L) -> list_to_binary(L); bin(B) when is_binary(B) -> B. --spec matches(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic(), [rule()]) -> +-spec matches(emqx_types:clientinfo(), action(), emqx_types:topic(), [rule()]) -> {matched, allow} | {matched, deny} | nomatch. -matches(_Client, _PubSub, _Topic, []) -> +matches(_Client, _Action, _Topic, []) -> nomatch; -matches(Client, PubSub, Topic, [{Permission, Who, Action, TopicFilters} | Tail]) -> - case match(Client, PubSub, Topic, {Permission, Who, Action, TopicFilters}) of - nomatch -> matches(Client, PubSub, Topic, Tail); +matches(Client, Action, Topic, [{Permission, WhoCond, ActionCond, TopicCond} | Tail]) -> + case match(Client, Action, Topic, {Permission, WhoCond, ActionCond, TopicCond}) of + nomatch -> matches(Client, Action, Topic, Tail); Matched -> Matched end. --spec match(emqx_types:clientinfo(), emqx_types:pubsub(), emqx_types:topic(), rule()) -> +-spec match(emqx_types:clientinfo(), action(), emqx_types:topic(), rule()) -> {matched, allow} | {matched, deny} | nomatch. -match(Client, PubSub, Topic, {Permission, Who, Action, TopicFilters}) -> +match(Client, Action, Topic, {Permission, WhoCond, ActionCond, TopicCond}) -> case - match_action(PubSub, Action) andalso - match_who(Client, Who) andalso - match_topics(Client, Topic, TopicFilters) + match_action(Action, ActionCond) andalso + match_who(Client, WhoCond) andalso + match_topics(Client, Topic, TopicCond) of true -> {matched, Permission}; _ -> nomatch end. -match_action(publish, publish) -> true; -match_action(subscribe, subscribe) -> true; -match_action(_, all) -> true; -match_action(_, _) -> false. +-spec match_action(action(), action_condition()) -> boolean(). +match_action(#{action_type := publish}, PubSubCond) when is_atom(PubSubCond) -> + match_pubsub(publish, PubSubCond); +match_action( + #{action_type := publish, qos := QoS, retain := Retain}, #{ + action_type := publish, qos := QoSCond, retain := RetainCond + } +) -> + match_qos(QoS, QoSCond) andalso match_retain(Retain, RetainCond); +match_action(#{action_type := publish, qos := QoS, retain := Retain}, #{ + action_type := all, qos := QoSCond, retain := RetainCond +}) -> + match_qos(QoS, QoSCond) andalso match_retain(Retain, RetainCond); +match_action(#{action_type := subscribe}, PubSubCond) when is_atom(PubSubCond) -> + match_pubsub(subscribe, PubSubCond); +match_action(#{action_type := subscribe, qos := QoS}, #{action_type := subscribe, qos := QoSCond}) -> + match_qos(QoS, QoSCond); +match_action(#{action_type := subscribe, qos := QoS}, #{action_type := all, qos := QoSCond}) -> + match_qos(QoS, QoSCond); +match_action(_, _) -> + false. + +match_pubsub(publish, publish) -> true; +match_pubsub(subscribe, subscribe) -> true; +match_pubsub(_, all) -> true; +match_pubsub(_, _) -> false. + +match_qos(QoS, QoSs) -> lists:member(QoS, QoSs). + +match_retain(_, all) -> true; +match_retain(Retain, Retain) when is_boolean(Retain) -> true; +match_retain(_, _) -> false. match_who(_, all) -> true; diff --git a/apps/emqx_authz/src/emqx_authz_rule_raw.erl b/apps/emqx_authz/src/emqx_authz_rule_raw.erl new file mode 100644 index 000000000..1fbe2ca45 --- /dev/null +++ b/apps/emqx_authz/src/emqx_authz_rule_raw.erl @@ -0,0 +1,197 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%% +%% @doc +%% This module converts authz rule fields obtained from +%% external sources like database or API to the format +%% accepted by emqx_authz_rule module. +%%-------------------------------------------------------------------- + +-module(emqx_authz_rule_raw). + +-export([parse_rule/1, format_rule/1]). + +-include("emqx_authz.hrl"). + +-type rule_raw() :: #{binary() => binary() | [binary()]}. + +%%-------------------------------------------------------------------- +%% API +%%-------------------------------------------------------------------- + +-spec parse_rule(rule_raw()) -> + {ok, { + emqx_authz_rule:permission(), + emqx_authz_rule:action_condition(), + emqx_authz_rule:topic_condition() + }} + | {error, term()}. +parse_rule( + #{ + <<"permission">> := PermissionRaw, + <<"action">> := ActionTypeRaw + } = RuleRaw +) -> + try + Topics = validate_rule_topics(RuleRaw), + Permission = validate_rule_permission(PermissionRaw), + ActionType = validate_rule_action_type(ActionTypeRaw), + Action = validate_rule_action(ActionType, RuleRaw), + {ok, {Permission, Action, Topics}} + catch + throw:ValidationError -> + {error, ValidationError} + end; +parse_rule(RuleRaw) -> + {error, {invalid_rule, RuleRaw}}. + +-spec format_rule({ + emqx_authz_rule:permission(), + emqx_authz_rule:action_condition(), + emqx_authz_rule:topic_condition() +}) -> map(). +format_rule({Permission, Action, Topics}) when is_list(Topics) -> + maps:merge( + #{ + topic => lists:map(fun format_topic/1, Topics), + permission => Permission + }, + format_action(Action) + ); +format_rule({Permission, Action, Topic}) -> + maps:merge( + #{ + topic => format_topic(Topic), + permission => Permission + }, + format_action(Action) + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +validate_rule_topics(#{<<"topic">> := TopicRaw}) when is_binary(TopicRaw) -> + [validate_rule_topic(TopicRaw)]; +validate_rule_topics(#{<<"topics">> := TopicsRaw}) when is_list(TopicsRaw) -> + lists:map(fun validate_rule_topic/1, TopicsRaw); +validate_rule_topics(RuleRaw) -> + throw({invalid_topics, RuleRaw}). + +validate_rule_topic(<<"eq ", TopicRaw/binary>>) -> + {eq, validate_rule_topic(TopicRaw)}; +validate_rule_topic(TopicRaw) when is_binary(TopicRaw) -> TopicRaw. + +validate_rule_permission(<<"allow">>) -> allow; +validate_rule_permission(<<"deny">>) -> deny; +validate_rule_permission(PermissionRaw) -> throw({invalid_permission, PermissionRaw}). + +validate_rule_action_type(<<"publish">>) -> publish; +validate_rule_action_type(<<"subscribe">>) -> subscribe; +validate_rule_action_type(<<"all">>) -> all; +validate_rule_action_type(ActionRaw) -> throw({invalid_action, ActionRaw}). + +validate_rule_action(ActionType, RuleRaw) -> + validate_rule_action(emqx_authz:feature_available(rich_actions), ActionType, RuleRaw). + +%% rich_actions disabled +validate_rule_action(false, ActionType, _RuleRaw) -> + ActionType; +%% rich_actions enabled +validate_rule_action(true, publish, RuleRaw) -> + Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)), + Retain = validate_rule_retain(maps:get(<<"retain">>, RuleRaw, <<"all">>)), + {publish, [{qos, Qos}, {retain, Retain}]}; +validate_rule_action(true, subscribe, RuleRaw) -> + Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)), + {subscribe, [{qos, Qos}]}; +validate_rule_action(true, all, RuleRaw) -> + Qos = validate_rule_qos(maps:get(<<"qos">>, RuleRaw, ?DEFAULT_RULE_QOS)), + Retain = validate_rule_retain(maps:get(<<"retain">>, RuleRaw, <<"all">>)), + {all, [{qos, Qos}, {retain, Retain}]}. + +validate_rule_qos(QosInt) when is_integer(QosInt) andalso QosInt >= 0 andalso QosInt =< 2 -> + [QosInt]; +validate_rule_qos(QosBin) when is_binary(QosBin) -> + try + QosRawList = binary:split(QosBin, <<",">>, [global]), + lists:map(fun validate_rule_qos_atomic/1, QosRawList) + catch + _:_ -> + throw({invalid_qos, QosBin}) + end; +validate_rule_qos(QosList) when is_list(QosList) -> + try + lists:map(fun validate_rule_qos_atomic/1, QosList) + catch + invalid_qos -> + throw({invalid_qos, QosList}) + end; +validate_rule_qos(undefined) -> + ?DEFAULT_RULE_QOS; +validate_rule_qos(null) -> + ?DEFAULT_RULE_QOS; +validate_rule_qos(QosRaw) -> + throw({invalid_qos, QosRaw}). + +validate_rule_qos_atomic(<<"0">>) -> 0; +validate_rule_qos_atomic(<<"1">>) -> 1; +validate_rule_qos_atomic(<<"2">>) -> 2; +validate_rule_qos_atomic(0) -> 0; +validate_rule_qos_atomic(1) -> 1; +validate_rule_qos_atomic(2) -> 2; +validate_rule_qos_atomic(_) -> throw(invalid_qos). + +validate_rule_retain(<<"0">>) -> false; +validate_rule_retain(<<"1">>) -> true; +validate_rule_retain(0) -> false; +validate_rule_retain(1) -> true; +validate_rule_retain(<<"true">>) -> true; +validate_rule_retain(<<"false">>) -> false; +validate_rule_retain(true) -> true; +validate_rule_retain(false) -> false; +validate_rule_retain(undefined) -> ?DEFAULT_RULE_RETAIN; +validate_rule_retain(null) -> ?DEFAULT_RULE_RETAIN; +validate_rule_retain(<<"all">>) -> ?DEFAULT_RULE_RETAIN; +validate_rule_retain(Retain) -> throw({invalid_retain, Retain}). + +format_action(Action) -> + format_action(emqx_authz:feature_available(rich_actions), Action). + +%% rich_actions disabled +format_action(false, Action) when is_atom(Action) -> + #{ + action => Action + }; +format_action(false, {ActionType, _Opts}) -> + #{ + action => ActionType + }; +%% rich_actions enabled +format_action(true, Action) when is_atom(Action) -> + #{ + action => Action + }; +format_action(true, {ActionType, Opts}) -> + #{ + action => ActionType, + qos => proplists:get_value(qos, Opts, ?DEFAULT_RULE_QOS), + retain => proplists:get_value(retain, Opts, ?DEFAULT_RULE_RETAIN) + }. + +format_topic({eq, Topic}) when is_binary(Topic) -> + <<"eq ", Topic/binary>>; +format_topic(Topic) when is_binary(Topic) -> + Topic. diff --git a/apps/emqx_authz/src/emqx_authz_utils.erl b/apps/emqx_authz/src/emqx_authz_utils.erl index ec112070e..d903ae027 100644 --- a/apps/emqx_authz/src/emqx_authz_utils.erl +++ b/apps/emqx_authz/src/emqx_authz_utils.erl @@ -31,7 +31,10 @@ parse_sql/3, render_deep/2, render_str/2, - render_sql_params/2 + render_sql_params/2, + client_vars/1, + vars_for_rule_query/2, + parse_rule_from_row/2 ]). -export([ @@ -43,6 +46,8 @@ start_after_created => false }). +-include_lib("emqx/include/logger.hrl"). + %%-------------------------------------------------------------------- %% APIs %%-------------------------------------------------------------------- @@ -171,6 +176,24 @@ content_type(Headers) when is_list(Headers) -> <<"application/json">> ). +-define(RAW_RULE_KEYS, [<<"permission">>, <<"action">>, <<"topic">>, <<"qos">>, <<"retain">>]). + +parse_rule_from_row(ColumnNames, Row) -> + RuleRaw = maps:with(?RAW_RULE_KEYS, maps:from_list(lists:zip(ColumnNames, to_list(Row)))), + case emqx_authz_rule_raw:parse_rule(RuleRaw) of + {ok, {Permission, Action, Topics}} -> + emqx_authz_rule:compile({Permission, all, Action, Topics}); + {error, Reason} -> + error(Reason) + end. + +vars_for_rule_query(Client, ?authz_action(PubSub, Qos) = Action) -> + Client#{ + action => PubSub, + qos => Qos, + retain => maps:get(retain, Action, false) + }. + %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- @@ -208,3 +231,8 @@ handle_sql_var(_Name, Value) -> bin(A) when is_atom(A) -> atom_to_binary(A, utf8); bin(L) when is_list(L) -> list_to_binary(L); bin(X) -> X. + +to_list(Tuple) when is_tuple(Tuple) -> + tuple_to_list(Tuple); +to_list(List) when is_list(List) -> + List. diff --git a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl index 3775b9a1c..7f03a38a2 100644 --- a/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl @@ -96,7 +96,7 @@ t_api(_) -> <<"hasnext">> := false } } = emqx_utils_json:decode(Request1), - ?assertEqual(3, length(Rules1)), + ?assertEqual(?USERNAME_RULES_EXAMPLE_COUNT, length(Rules1)), {ok, 200, Request1_1} = request( @@ -204,7 +204,7 @@ t_api(_) -> } = emqx_utils_json:decode(Request4), #{<<"clientid">> := <<"client1">>, <<"rules">> := Rules3} = emqx_utils_json:decode(Request5), - ?assertEqual(3, length(Rules3)), + ?assertEqual(?CLIENTID_RULES_EXAMPLE_COUNT, length(Rules3)), {ok, 204, _} = request( @@ -253,7 +253,7 @@ t_api(_) -> [] ), #{<<"rules">> := Rules5} = emqx_utils_json:decode(Request7), - ?assertEqual(3, length(Rules5)), + ?assertEqual(?ALL_RULES_EXAMPLE_COUNT, length(Rules5)), {ok, 204, _} = request( diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl index ec96522a5..0ce788f8d 100644 --- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl @@ -42,11 +42,11 @@ init_per_suite(Config) -> Config. end_per_suite(_Config) -> - ok. + ok = emqx_authz_test_lib:restore_authorizers(). init_per_testcase(TestCase, Config) -> Apps = emqx_cth_suite:start( - [{emqx_conf, "authorization.no_match = deny"}, emqx_authz], + [{emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"}, emqx_authz], #{work_dir => filename:join(?config(priv_dir, Config), TestCase)} ), [{tc_apps, Apps} | Config]. @@ -59,13 +59,7 @@ end_per_testcase(_TestCase, Config) -> %%------------------------------------------------------------------------------ t_ok(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ClientInfo = emqx_authz_test_lib:base_client_info(), ok = setup_config(?RAW_SOURCE#{ <<"rules">> => <<"{allow, {user, \"username\"}, publish, [\"t\"]}.">> @@ -73,23 +67,52 @@ t_ok(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), ?assertEqual( deny, - emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>) + ). + +t_rich_actions(_Config) -> + ClientInfo = emqx_authz_test_lib:base_client_info(), + + ok = setup_config(?RAW_SOURCE#{ + <<"rules">> => + <<"{allow, {user, \"username\"}, {publish, [{qos, 1}, {retain, false}]}, [\"t\"]}.">> + }), + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>) + ), + + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(0, false), <<"t">>) + ), + + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>) + ). + +t_no_rich_actions(_Config) -> + _ = emqx_authz:set_feature_available(rich_actions, false), + ?assertMatch( + {error, {pre_config_update, emqx_authz, {invalid_authorization_action, _}}}, + emqx_authz:update(?CMD_REPLACE, [ + ?RAW_SOURCE#{ + <<"rules">> => + <<"{allow, {user, \"username\"}, {publish, [{qos, 1}, {retain, false}]}, [\"t\"]}.">> + } + ]) ). t_superuser(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - is_superuser => true, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ClientInfo = + emqx_authz_test_lib:client_info(#{is_superuser => true}), %% no rules apply to superuser ok = setup_config(?RAW_SOURCE#{ @@ -98,12 +121,12 @@ t_superuser(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, subscribe, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>) ). t_invalid_file(_Config) -> diff --git a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl index 702bf2756..6cf4b5bc0 100644 --- a/apps/emqx_authz/test/emqx_authz_http_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_http_SUITE.erl @@ -65,6 +65,7 @@ init_per_testcase(_Case, Config) -> Config. end_per_testcase(_Case, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), try ok = emqx_authz_http_test_server:stop() catch @@ -97,7 +98,7 @@ t_response_handling(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% Not OK, get, no body @@ -109,7 +110,7 @@ t_response_handling(_Config) -> #{} ), - deny = emqx_access_control:authorize(ClientInfo, publish, <<"t">>), + deny = emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>), %% OK, get, 204 ok = setup_handler_and_config( @@ -122,7 +123,7 @@ t_response_handling(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% Not OK, get, 400 @@ -136,7 +137,7 @@ t_response_handling(_Config) -> ?assertEqual( deny, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% Not OK, get, 400 + body & headers @@ -155,7 +156,7 @@ t_response_handling(_Config) -> ?assertEqual( deny, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% the server cannot be reached; should skip to the next @@ -165,7 +166,7 @@ t_response_handling(_Config) -> ?check_trace( ?assertEqual( deny, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), fun(Trace) -> ?assertMatch( @@ -200,7 +201,9 @@ t_query_params(_Config) -> proto_name := <<"MQTT">>, mountpoint := <<"MOUNTPOINT">>, topic := <<"t/1">>, - action := <<"publish">> + action := <<"publish">>, + qos := <<"1">>, + retain := <<"false">> } = cowboy_req:match_qs( [ username, @@ -209,7 +212,9 @@ t_query_params(_Config) -> proto_name, mountpoint, topic, - action + action, + qos, + retain ], Req0 ), @@ -224,7 +229,9 @@ t_query_params(_Config) -> "proto_name=${proto_name}&" "mountpoint=${mountpoint}&" "topic=${topic}&" - "action=${action}" + "action=${action}&" + "qos=${qos}&" + "retain=${retain}" >> } ), @@ -241,7 +248,7 @@ t_query_params(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t/1">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t/1">>) ). t_path(_Config) -> @@ -256,7 +263,9 @@ t_path(_Config) -> "MQTT/" "MOUNTPOINT/" "t%2F1/" - "publish" + "publish/" + "1/" + "false" >>, cowboy_req:path(Req0) ), @@ -271,7 +280,9 @@ t_path(_Config) -> "${proto_name}/" "${mountpoint}/" "${topic}/" - "${action}" + "${action}/" + "${qos}/" + "${retain}" >> } ), @@ -288,7 +299,7 @@ t_path(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t/1">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t/1">>) ). t_json_body(_Config) -> @@ -309,7 +320,9 @@ t_json_body(_Config) -> <<"proto_name">> := <<"MQTT">>, <<"mountpoint">> := <<"MOUNTPOINT">>, <<"topic">> := <<"t">>, - <<"action">> := <<"publish">> + <<"action">> := <<"publish">>, + <<"qos">> := <<"1">>, + <<"retain">> := <<"false">> }, emqx_utils_json:decode(RawBody, [return_maps]) ), @@ -324,7 +337,9 @@ t_json_body(_Config) -> <<"proto_name">> => <<"${proto_name}">>, <<"mountpoint">> => <<"${mountpoint}">>, <<"topic">> => <<"${topic}">>, - <<"action">> => <<"${action}">> + <<"action">> => <<"${action}">>, + <<"qos">> => <<"${qos}">>, + <<"retain">> => <<"${retain}">> } } ), @@ -341,7 +356,45 @@ t_json_body(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>) + ). + +t_no_rich_actions(_Config) -> + _ = emqx_authz:set_feature_available(rich_actions, false), + + ok = setup_handler_and_config( + fun(Req0, State) -> + ?assertEqual( + <<"/authz/users/">>, + cowboy_req:path(Req0) + ), + + {ok, RawBody, Req1} = cowboy_req:read_body(Req0), + + %% No interpolation if rich_actions is disabled + ?assertMatch( + #{ + <<"qos">> := <<"${qos}">>, + <<"retain">> := <<"${retain}">> + }, + emqx_utils_json:decode(RawBody, [return_maps]) + ), + {ok, ?AUTHZ_HTTP_RESP(allow, Req1), State} + end, + #{ + <<"method">> => <<"post">>, + <<"body">> => #{ + <<"qos">> => <<"${qos}">>, + <<"retain">> => <<"${retain}">> + } + } + ), + + ClientInfo = emqx_authz_test_lib:base_client_info(), + + ?assertEqual( + allow, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>) ). t_placeholder_and_body(_Config) -> @@ -401,7 +454,7 @@ t_placeholder_and_body(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). t_no_value_for_placeholder(_Config) -> @@ -441,7 +494,7 @@ t_no_value_for_placeholder(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). t_create_replace(_Config) -> @@ -466,7 +519,7 @@ t_create_replace(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), %% Changing to valid config @@ -485,7 +538,7 @@ t_create_replace(_Config) -> ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl b/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl index 56e893c5b..fcaa378c5 100644 --- a/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_jwt_SUITE.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx_placeholder.hrl"). -include_lib("emqx_authn/include/emqx_authn.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -341,12 +342,12 @@ t_check_undefined_expire(_Config) -> ?assertMatch( {matched, allow}, - emqx_authz_client_info:authorize(Client, subscribe, <<"a/b">>, undefined) + emqx_authz_client_info:authorize(Client, ?AUTHZ_SUBSCRIBE, <<"a/b">>, undefined) ), ?assertMatch( {matched, deny}, - emqx_authz_client_info:authorize(Client, subscribe, <<"a/bar">>, undefined) + emqx_authz_client_info:authorize(Client, ?AUTHZ_SUBSCRIBE, <<"a/bar">>, undefined) ). %%------------------------------------------------------------------------------ diff --git a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl index 2b7fce309..c82bbd56e 100644 --- a/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl @@ -18,6 +18,8 @@ -compile(nowarn_export_all). -compile(export_all). +-include_lib("emqx_authz.hrl"). + -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -44,6 +46,7 @@ init_per_testcase(_TestCase, Config) -> Config. end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), ok = emqx_authz_mnesia:purge_rules(). set_special_configs(emqx_authz) -> @@ -54,51 +57,135 @@ set_special_configs(_) -> %%------------------------------------------------------------------------------ %% Testcases %%------------------------------------------------------------------------------ -t_username_topic_rules(_Config) -> - ok = test_topic_rules(username). -t_clientid_topic_rules(_Config) -> - ok = test_topic_rules(clientid). +t_authz(_Config) -> + ClientInfo = emqx_authz_test_lib:base_client_info(), -t_all_topic_rules(_Config) -> - ok = test_topic_rules(all). + test_authz( + allow, + allow, + {all, #{ + <<"permission">> => <<"allow">>, <<"action">> => <<"subscribe">>, <<"topic">> => <<"t">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t">>} + ), + test_authz( + allow, + allow, + {{username, <<"username">>}, #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"t/${username}">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/username">>} + ), + test_authz( + allow, + allow, + {{username, <<"username">>}, #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"eq t/${username}">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/${username}">>} + ), + test_authz( + deny, + deny, + {{username, <<"username">>}, #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"eq t/${username}">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/username">>} + ), + test_authz( + allow, + allow, + {{clientid, <<"clientid">>}, #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"eq t/${username}">> + }}, + {ClientInfo, ?AUTHZ_SUBSCRIBE, <<"t/${username}">>} + ), + test_authz( + allow, + allow, + { + {clientid, <<"clientid">>}, + #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"t">>, + <<"qos">> => <<"1,2">>, + <<"retain">> => <<"true">> + } + }, + {ClientInfo, ?AUTHZ_PUBLISH(1, true), <<"t">>} + ), + test_authz( + deny, + allow, + { + {clientid, <<"clientid">>}, + #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"t">>, + <<"qos">> => <<"1,2">>, + <<"retain">> => <<"true">> + } + }, + {ClientInfo, ?AUTHZ_PUBLISH(0, true), <<"t">>} + ), + test_authz( + deny, + allow, + { + {clientid, <<"clientid">>}, + #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"t">>, + <<"qos">> => <<"1,2">>, + <<"retain">> => <<"true">> + } + }, + {ClientInfo, ?AUTHZ_PUBLISH(1, false), <<"t">>} + ). -test_topic_rules(Key) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, +test_authz(Expected, ExpectedNoRichActions, {Who, Rule}, {ClientInfo, Action, Topic}) -> + test_authz_with_rich_actions(true, Expected, {Who, Rule}, {ClientInfo, Action, Topic}), + test_authz_with_rich_actions( + false, ExpectedNoRichActions, {Who, Rule}, {ClientInfo, Action, Topic} + ). - SetupSamples = fun(CInfo, Samples) -> - setup_client_samples(CInfo, Samples, Key) - end, - - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, SetupSamples), - - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, SetupSamples), - - ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, SetupSamples). +test_authz_with_rich_actions( + RichActionsEnabled, Expected, {Who, Rule}, {ClientInfo, Action, Topic} +) -> + ct:pal("Test authz rich_actions:~p~nwho:~p~nrule:~p~nattempt:~p~nexpected ~p", [ + RichActionsEnabled, Who, Rule, {ClientInfo, Action, Topic}, Expected + ]), + try + _ = emqx_authz:set_feature_available(rich_actions, RichActionsEnabled), + ok = emqx_authz_mnesia:store_rules(Who, [Rule]), + ?assertEqual(Expected, emqx_access_control:authorize(ClientInfo, Action, Topic)) + after + ok = emqx_authz_mnesia:purge_rules() + end. t_normalize_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ClientInfo = emqx_authz_test_lib:base_client_info(), ok = emqx_authz_mnesia:store_rules( {username, <<"username">>}, - [{allow, publish, "t"}] + [#{<<"permission">> => <<"allow">>, <<"action">> => <<"publish">>, <<"topic">> => <<"t">>}] ), ?assertEqual( allow, - emqx_access_control:authorize(ClientInfo, publish, <<"t">>) + emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>) ), ?assertException( @@ -106,25 +193,31 @@ t_normalize_rules(_Config) -> {invalid_rule, _}, emqx_authz_mnesia:store_rules( {username, <<"username">>}, - [[allow, publish, <<"t">>]] + [[<<"allow">>, <<"publish">>, <<"t">>]] ) ), ?assertException( error, - {invalid_rule_action, _}, + {invalid_action, _}, emqx_authz_mnesia:store_rules( {username, <<"username">>}, - [{allow, pub, <<"t">>}] + [#{<<"permission">> => <<"allow">>, <<"action">> => <<"pub">>, <<"topic">> => <<"t">>}] ) ), ?assertException( error, - {invalid_rule_permission, _}, + {invalid_permission, _}, emqx_authz_mnesia:store_rules( {username, <<"username">>}, - [{accept, publish, <<"t">>}] + [ + #{ + <<"permission">> => <<"accept">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"t">> + } + ] ) ). @@ -138,27 +231,5 @@ raw_mnesia_authz_config() -> <<"type">> => <<"built_in_database">> }. -setup_client_samples(ClientInfo, Samples, Key) -> - ok = emqx_authz_mnesia:purge_rules(), - Rules = lists:flatmap( - fun(#{topics := Topics, permission := Permission, action := Action}) -> - lists:map( - fun(Topic) -> - {binary_to_atom(Permission), binary_to_atom(Action), Topic} - end, - Topics - ) - end, - Samples - ), - #{username := Username, clientid := ClientId} = ClientInfo, - Who = - case Key of - username -> {username, Username}; - clientid -> {clientid, ClientId}; - all -> all - end, - ok = emqx_authz_mnesia:store_rules(Who, Rules). - setup_config() -> emqx_authz_test_lib:setup_config(raw_mnesia_authz_config(), #{}). diff --git a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl index 9ffeacf45..4476deda2 100644 --- a/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl @@ -28,10 +28,10 @@ -define(MONGO_CLIENT, 'emqx_authz_mongo_SUITE_client'). all() -> - emqx_common_test_helpers:all(?MODULE). + emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()). groups() -> - []. + emqx_authz_test_lib:table_groups(t_run_case, cases()). init_per_suite(Config) -> ok = stop_apps([emqx_resource]), @@ -57,12 +57,18 @@ set_special_configs(emqx_authz) -> set_special_configs(_) -> ok. +init_per_group(Group, Config) -> + [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config]. +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> {ok, _} = mc_worker_api:connect(mongo_config()), ok = emqx_authz_test_lib:reset_authorizers(), Config. end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), ok = reset_samples(), ok = mc_worker_api:disconnect(?MONGO_CLIENT). @@ -70,233 +76,313 @@ end_per_testcase(_TestCase, _Config) -> %% Testcases %%------------------------------------------------------------------------------ -t_topic_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, +t_run_case(Config) -> + Case = ?config(test_case, Config), + ok = setup_source_data(Case), + ok = setup_authz_source(Case), + ok = emqx_authz_test_lib:run_checks(Case). - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2), +%%------------------------------------------------------------------------------ +%% Cases +%%------------------------------------------------------------------------------ - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2). - -t_complex_filter(_) -> - %% atom and string values also supported - ClientInfo = #{ - clientid => clientid, - username => "username", - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - Samples = [ +cases() -> + [ #{ - <<"x">> => #{ - <<"u">> => <<"username">>, - <<"c">> => [#{<<"c">> => <<"clientid">>}], - <<"y">> => 1 - }, - <<"permission">> => <<"allow">>, - <<"action">> => <<"publish">>, - <<"topics">> => [<<"t">>] - } - ], - - ok = setup_samples(Samples), - ok = setup_config( - #{ - <<"filter">> => #{ - <<"x">> => #{ - <<"u">> => <<"${username}">>, - <<"c">> => [#{<<"c">> => <<"${clientid}">>}], - <<"y">> => 1 + name => base_publish, + records => [ + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"subscribe">>, + <<"topic">> => <<"b">>, + <<"permission">> => <<"allow">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"all">>, + <<"topics">> => [<<"c">>, <<"d">>], + <<"permission">> => <<"allow">> } - } + ], + filter => #{<<"username">> => <<"${username}">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"a">>}, + + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"b">>}, + + {allow, ?AUTHZ_PUBLISH, <<"c">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"c">>}, + {allow, ?AUTHZ_PUBLISH, <<"d">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"d">>} + ] + }, + #{ + name => filter_works, + records => [ + #{ + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + filter => #{<<"username">> => <<"${username}">>}, + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_rich_rules, + features => [rich_actions], + records => [ + #{ + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">>, + <<"qos">> => <<"1,2,3">> + }, + #{ + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">>, + <<"retain">> => <<"yes">> + } + ], + filter => #{}, + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_rules, + records => [ + #{ + <<"action">> => <<"publis">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + filter => #{}, + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => rule_by_clientid_cn_dn_peerhost, + records => [ + #{ + <<"cn">> => <<"cn">>, + <<"dn">> => <<"dn">>, + <<"clientid">> => <<"clientid">>, + <<"peerhost">> => <<"127.0.0.1">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + client_info => #{ + cn => <<"cn">>, + dn => <<"dn">> + }, + filter => #{ + <<"cn">> => <<"${cert_common_name}">>, + <<"dn">> => <<"${cert_subject}">>, + <<"clientid">> => <<"${clientid}">>, + <<"peerhost">> => <<"${peerhost}">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => topics_literal_wildcard_variable, + records => [ + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topics">> => [ + <<"t/${username}">>, + <<"t/${clientid}">>, + <<"t1/#">>, + <<"t2/+">>, + <<"eq t3/${username}">> + ] + } + ], + filter => #{<<"username">> => <<"${username}">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {allow, ?AUTHZ_PUBLISH, <<"t1/a/b">>}, + {allow, ?AUTHZ_PUBLISH, <<"t2/a">>}, + {allow, ?AUTHZ_PUBLISH, <<"t3/${username}">>}, + {deny, ?AUTHZ_PUBLISH, <<"t3/username">>} + ] + }, + #{ + name => qos_retain_in_query_result, + features => [rich_actions], + records => [ + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"a">>, + <<"qos">> => 1, + <<"retain">> => true + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"b">>, + <<"qos">> => <<"1">>, + <<"retain">> => <<"true">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"c">>, + <<"qos">> => <<"1,2">>, + <<"retain">> => 1 + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"d">>, + <<"qos">> => [1, 2], + <<"retain">> => <<"1">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"e">>, + <<"qos">> => [1, 2], + <<"retain">> => <<"all">> + }, + #{ + <<"username">> => <<"username">>, + <<"action">> => <<"publish">>, + <<"permission">> => <<"allow">>, + <<"topic">> => <<"f">>, + <<"qos">> => null, + <<"retain">> => null + } + ], + filter => #{<<"username">> => <<"${username}">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"a">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"a">>}, + + {allow, ?AUTHZ_PUBLISH(1, true), <<"b">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"b">>}, + {deny, ?AUTHZ_PUBLISH(2, false), <<"b">>}, + + {allow, ?AUTHZ_PUBLISH(2, true), <<"c">>}, + {deny, ?AUTHZ_PUBLISH(2, false), <<"c">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"c">>}, + + {allow, ?AUTHZ_PUBLISH(2, true), <<"d">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"d">>}, + + {allow, ?AUTHZ_PUBLISH(1, false), <<"e">>}, + {allow, ?AUTHZ_PUBLISH(1, true), <<"e">>}, + {deny, ?AUTHZ_PUBLISH(0, false), <<"e">>}, + + {allow, ?AUTHZ_PUBLISH, <<"f">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"f">>} + ] + }, + #{ + name => nonbin_values_in_client_info, + records => [ + #{ + <<"username">> => <<"username">>, + <<"clientid">> => <<"clientid">>, + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + client_info => #{ + username => "username", + clientid => clientid + }, + filter => #{<<"username">> => <<"${username}">>, <<"clientid">> => <<"${clientid}">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_query, + records => [ + #{ + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + filter => #{<<"$in">> => #{<<"a">> => 1}}, + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => complex_query, + records => [ + #{ + <<"a">> => #{<<"u">> => <<"clientid">>, <<"c">> => [<<"cn">>, <<"dn">>]}, + <<"action">> => <<"publish">>, + <<"topic">> => <<"a">>, + <<"permission">> => <<"allow">> + } + ], + client_info => #{ + cn => <<"cn">>, + dn => <<"dn">> + }, + filter => #{ + <<"a">> => #{ + <<"u">> => <<"${clientid}">>, + <<"c">> => [<<"${cert_common_name}">>, <<"${cert_subject}">>] + } + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [{allow, publish, <<"t">>}] - ). - -t_mongo_error(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = setup_samples([]), - ok = setup_config( - #{<<"filter">> => #{<<"$badoperator">> => <<"$badoperator">>}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [{deny, publish, <<"t">>}] - ). - -t_lookups(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ByClientid = #{ - <<"clientid">> => <<"clientid">>, - <<"topics">> => [<<"a">>], - <<"action">> => <<"all">>, - <<"permission">> => <<"allow">> - }, - - ok = setup_samples([ByClientid]), - ok = setup_config( - #{<<"filter">> => #{<<"clientid">> => <<"${clientid}">>}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByPeerhost = #{ - <<"peerhost">> => <<"127.0.0.1">>, - <<"topics">> => [<<"a">>], - <<"action">> => <<"all">>, - <<"permission">> => <<"allow">> - }, - - ok = setup_samples([ByPeerhost]), - ok = setup_config( - #{<<"filter">> => #{<<"peerhost">> => <<"${peerhost}">>}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByCN = #{ - <<"CN">> => <<"cn">>, - <<"topics">> => [<<"a">>], - <<"action">> => <<"all">>, - <<"permission">> => <<"allow">> - }, - - ok = setup_samples([ByCN]), - ok = setup_config( - #{<<"filter">> => #{<<"CN">> => ?PH_CERT_CN_NAME}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByDN = #{ - <<"DN">> => <<"dn">>, - <<"topics">> => [<<"a">>], - <<"action">> => <<"all">>, - <<"permission">> => <<"allow">> - }, - - ok = setup_samples([ByDN]), - ok = setup_config( - #{<<"filter">> => #{<<"DN">> => ?PH_CERT_SUBJECT}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). - -t_bad_filter(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = setup_config( - #{<<"filter">> => #{<<"$in">> => #{<<"a">> => 1}}} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {deny, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). + ]. %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ -populate_records(AclRecords, AdditionalData) -> - [maps:merge(Record, AdditionalData) || Record <- AclRecords]. - -setup_samples(AclRecords) -> - ok = reset_samples(), - {{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"acl">>, AclRecords), - ok. - -setup_client_samples(ClientInfo, Samples) -> - #{username := Username} = ClientInfo, - Records = lists:map( - fun(Sample) -> - #{ - topics := Topics, - permission := Permission, - action := Action - } = Sample, - - #{ - <<"topics">> => Topics, - <<"permission">> => Permission, - <<"action">> => Action, - <<"username">> => Username - } - end, - Samples - ), - setup_samples(Records), - setup_config(#{<<"filter">> => #{<<"username">> => <<"${username}">>}}). - reset_samples() -> {true, _} = mc_worker_api:delete(?MONGO_CLIENT, <<"acl">>, #{}), ok. +setup_source_data(#{records := Records}) -> + {{true, _}, _} = mc_worker_api:insert(?MONGO_CLIENT, <<"acl">>, Records), + ok. + +setup_authz_source(#{filter := Filter}) -> + setup_config( + #{ + <<"filter">> => Filter + } + ). + setup_config(SpecialParams) -> emqx_authz_test_lib:setup_config( raw_mongo_authz_config(), diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index 06449b3b4..f31a6ceab 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -27,10 +27,10 @@ -define(MYSQL_RESOURCE, <<"emqx_authz_mysql_SUITE">>). all() -> - emqx_common_test_helpers:all(?MODULE). + emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()). groups() -> - []. + emqx_authz_test_lib:table_groups(t_run_case, cases()). init_per_suite(Config) -> ok = stop_apps([emqx_resource]), @@ -41,13 +41,7 @@ init_per_suite(Config) -> fun set_special_configs/1 ), ok = start_apps([emqx_resource]), - {ok, _} = emqx_resource:create_local( - ?MYSQL_RESOURCE, - ?RESOURCE_GROUP, - emqx_mysql, - mysql_config(), - #{} - ), + ok = create_mysql_resource(), Config; false -> {skip, no_mysql} @@ -59,9 +53,18 @@ end_per_suite(_Config) -> ok = stop_apps([emqx_resource]), ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). +init_per_group(Group, Config) -> + [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config]. +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ok = drop_table(), + ok. set_special_configs(emqx_authz) -> ok = emqx_authz_test_lib:reset_authorizers(); @@ -72,189 +75,11 @@ set_special_configs(_) -> %% Testcases %%------------------------------------------------------------------------------ -t_topic_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2). - -t_lookups(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - %% by clientid - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(clientid, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${clientid}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by peerhost - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(peerhost, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"127.0.0.1">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE peerhost = ${peerhost}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by cn - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(cn, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"cn">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE cn = ${cert_common_name}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by dn - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(dn, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"dn">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE dn = ${cert_subject}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% strip double quote support - - ok = init_table(), - ok = q( - << - "INSERT INTO acl(clientid, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = \"${clientid}\"" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). - -t_mysql_error(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = setup_config( - #{<<"query">> => <<"SOME INVALID STATEMENT">>} - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [{deny, subscribe, <<"a">>}] - ). +t_run_case(Config) -> + Case = ?config(test_case, Config), + ok = setup_source_data(Case), + ok = setup_authz_source(Case), + ok = emqx_authz_test_lib:run_checks(Case). t_create_invalid(_Config) -> BadConfig = maps:merge( @@ -265,45 +90,285 @@ t_create_invalid(_Config) -> [_] = emqx_authz:lookup(). -t_nonbinary_values(_Config) -> - ClientInfo = #{ - clientid => clientid, - username => "username", - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, +%%------------------------------------------------------------------------------ +%% Cases +%%------------------------------------------------------------------------------ - ok = init_table(), - ok = q( - << - "INSERT INTO acl(clientid, username, topic, permission, action)" - "VALUES(?, ?, ?, ?, ?)" - >>, - [<<"clientid">>, <<"username">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( +cases() -> + [ #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${clientid} AND username = ${username}" - >> - } - ), + name => base_publish, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'b', 'allow', 'subscribe')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + client_info => #{username => <<"username">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"a">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"b">>} + ] + }, + #{ + name => rule_by_clientid_cn_dn_peerhost, + setup => [ + "CREATE TABLE acl(clientid VARCHAR(255), cn VARCHAR(255), dn VARCHAR(255)," + " peerhost VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))", - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). + "INSERT INTO acl(clientid, cn, dn, peerhost, topic, permission, action)" + " VALUES('clientid', 'cn', 'dn', '127.0.0.1', 'a', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic FROM acl WHERE" + " clientid = ${clientid} AND cn = ${cert_common_name}" + " AND dn = ${cert_subject} AND peerhost = ${peerhost}", + client_info => #{ + clientid => <<"clientid">>, + cn => <<"cn">>, + dn => <<"dn">>, + peerhost => {127, 0, 0, 1} + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>} + ] + }, + #{ + name => topics_literal_wildcard_variable, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/${username}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/${clientid}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 'eq t/${username}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/#', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't1/+', 'allow', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/${username}">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/1/2">>}, + {allow, ?AUTHZ_PUBLISH, <<"t1/1">>}, + {deny, ?AUTHZ_PUBLISH, <<"t1/1/2">>}, + {deny, ?AUTHZ_PUBLISH, <<"abc">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"t/username">>} + ] + }, + #{ + name => qos_retain_in_query_result, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255)," + "qos_s VARCHAR(255), retain_s VARCHAR(255))", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't1', 'allow', 'publish', '1', 'true')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't2', 'allow', 'publish', '2', 'false')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't3', 'allow', 'publish', '0,1,2', 'all')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't4', 'allow', 'subscribe', '1', null)", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't5', 'allow', 'subscribe', '0,1,2', null)" + ], + query => + "SELECT permission, action, topic, qos_s as qos, retain_s as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>}, + + {allow, ?AUTHZ_PUBLISH(2, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(2, true), <<"t2">>}, + + {allow, ?AUTHZ_PUBLISH(1, true), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(2, false), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(2, true), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(0, false), <<"t3">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"t4">>}, + {deny, ?AUTHZ_SUBSCRIBE(2), <<"t4">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"t5">>}, + {allow, ?AUTHZ_SUBSCRIBE(2), <<"t5">>}, + {allow, ?AUTHZ_SUBSCRIBE(0), <<"t5">>} + ] + }, + #{ + name => qos_retain_in_query_result_as_integer, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255)," + "qos_i VARCHAR(255), retain_i VARCHAR(255))", + + "INSERT INTO acl(username, topic, permission, action, qos_i, retain_i)" + " VALUES('username', 't1', 'allow', 'publish', 1, 1)" + ], + query => + "SELECT permission, action, topic, qos_i as qos, retain_i as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>} + ] + }, + #{ + name => retain_in_query_result_as_boolean, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255)," + " action VARCHAR(255), retain_b BOOLEAN)", + + "INSERT INTO acl(username, topic, permission, action, retain_b)" + " VALUES('username', 't1', 'allow', 'publish', true)", + + "INSERT INTO acl(username, topic, permission, action, retain_b)" + " VALUES('username', 't2', 'allow', 'publish', false)" + ], + query => + "SELECT permission, action, topic, retain_b as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {allow, ?AUTHZ_PUBLISH(1, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(1, true), <<"t2">>} + ] + }, + #{ + name => nonbin_values_in_client_info, + setup => [ + "CREATE TABLE acl(who VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255)," + " action VARCHAR(255))", + + "INSERT INTO acl(who, topic, permission, action)" + " VALUES('username', 't/${username}', 'allow', 'publish')", + + "INSERT INTO acl(who, topic, permission, action)" + " VALUES('clientid', 't/${clientid}', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic" + " FROM acl WHERE who = ${username} OR who = ${clientid}", + client_info => #{ + %% string, not a binary + username => "username", + %% atom, not a binary + clientid => clientid + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {deny, ?AUTHZ_PUBLISH, <<"t/foo">>} + ] + }, + #{ + name => null_retain_qos, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(qos VARCHAR(255), retain VARCHAR(255)," + " topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))", + + "INSERT INTO acl(qos, retain, topic, permission, action)" + " VALUES(NULL, NULL, 'tp', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic, qos FROM acl", + checks => [ + {allow, ?AUTHZ_PUBLISH(0, false), <<"tp">>}, + {allow, ?AUTHZ_PUBLISH(1, false), <<"tp">>}, + {allow, ?AUTHZ_PUBLISH(2, true), <<"tp">>}, + + {deny, ?AUTHZ_PUBLISH(0, true), <<"xxx">>} + ] + }, + #{ + name => strip_double_quote, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = \"${username}\"", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_query, + setup => [], + query => "SELECT permission, action, topic FROM acl WHER", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => pgsql_error, + setup => [], + query => + "SELECT permission, action, topic FROM table_not_exists WHERE username = ${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"t">>} + ] + } + ]. %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ +setup_source_data(#{setup := Queries}) -> + lists:foreach( + fun(Query) -> + _ = q(Query) + end, + Queries + ). + +setup_authz_source(#{query := Query}) -> + setup_config( + #{ + <<"query">> => Query + } + ). + raw_mysql_authz_config() -> #{ <<"enable">> => <<"true">>, @@ -332,52 +397,9 @@ q(Sql, Params) -> {sql, Sql, Params} ). -init_table() -> - ok = drop_table(), - ok = q( - "CREATE TABLE acl(\n" - " username VARCHAR(255),\n" - " clientid VARCHAR(255),\n" - " peerhost VARCHAR(255),\n" - " cn VARCHAR(255),\n" - " dn VARCHAR(255),\n" - " topic VARCHAR(255),\n" - " permission VARCHAR(255),\n" - " action VARCHAR(255))" - ). - drop_table() -> ok = q("DROP TABLE IF EXISTS acl"). -setup_client_samples(ClientInfo, Samples) -> - #{username := Username} = ClientInfo, - ok = init_table(), - ok = lists:foreach( - fun(#{topics := Topics, permission := Permission, action := Action}) -> - lists:foreach( - fun(Topic) -> - q( - << - "INSERT INTO acl(username, topic, permission, action)" - "VALUES(?, ?, ?, ?)" - >>, - [Username, Topic, Permission, Action] - ) - end, - Topics - ) - end, - Samples - ), - setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE username = ${username}" - >> - } - ). - setup_config(SpecialParams) -> emqx_authz_test_lib:setup_config( raw_mysql_authz_config(), @@ -400,3 +422,13 @@ start_apps(Apps) -> stop_apps(Apps) -> lists:foreach(fun application:stop/1, Apps). + +create_mysql_resource() -> + {ok, _} = emqx_resource:create_local( + ?MYSQL_RESOURCE, + ?RESOURCE_GROUP, + emqx_mysql, + mysql_config(), + #{} + ), + ok. diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl index 0ef21360c..0c446ee99 100644 --- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl @@ -27,10 +27,10 @@ -define(PGSQL_RESOURCE, <<"emqx_authz_pgsql_SUITE">>). all() -> - emqx_common_test_helpers:all(?MODULE). + emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()). groups() -> - []. + emqx_authz_test_lib:table_groups(t_run_case, cases()). init_per_suite(Config) -> ok = stop_apps([emqx_resource]), @@ -41,13 +41,7 @@ init_per_suite(Config) -> fun set_special_configs/1 ), ok = start_apps([emqx_resource]), - {ok, _} = emqx_resource:create_local( - ?PGSQL_RESOURCE, - ?RESOURCE_GROUP, - emqx_connector_pgsql, - pgsql_config(), - #{} - ), + {ok, _} = create_pgsql_resource(), Config; false -> {skip, no_pgsql} @@ -59,9 +53,18 @@ end_per_suite(_Config) -> ok = stop_apps([emqx_resource]), ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). +init_per_group(Group, Config) -> + [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config]. +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ok = drop_table(), + ok. set_special_configs(emqx_authz) -> ok = emqx_authz_test_lib:reset_authorizers(); @@ -72,194 +75,11 @@ set_special_configs(_) -> %% Testcases %%------------------------------------------------------------------------------ -t_topic_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_deny_topic_rules(ClientInfo, fun setup_client_samples/2). - -t_lookups(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - %% by clientid - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(clientid, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${clientid}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by peerhost - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(peerhost, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"127.0.0.1">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE peerhost = ${peerhost}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by cn - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(cn, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"cn">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE cn = ${cert_common_name}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% by dn - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(dn, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"dn">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE dn = ${cert_subject}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - %% strip double quote support - - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(clientid, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [<<"clientid">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = \"${clientid}\"" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). - -t_pgsql_error(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${username}" - >> - } - ), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [{deny, subscribe, <<"a">>}] - ). +t_run_case(Config) -> + Case = ?config(test_case, Config), + ok = setup_source_data(Case), + ok = setup_authz_source(Case), + ok = emqx_authz_test_lib:run_checks(Case). t_create_invalid(_Config) -> BadConfig = maps:merge( @@ -270,45 +90,291 @@ t_create_invalid(_Config) -> [_] = emqx_authz:lookup(). -t_nonbinary_values(_Config) -> - ClientInfo = #{ - clientid => clientid, - username => "username", - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, +%%------------------------------------------------------------------------------ +%% Cases +%%------------------------------------------------------------------------------ - ok = init_table(), - ok = insert( - << - "INSERT INTO acl(clientid, username, topic, permission, action)" - "VALUES($1, $2, $3, $4, $5)" - >>, - [<<"clientid">>, <<"username">>, <<"a">>, <<"allow">>, <<"subscribe">>] - ), - - ok = setup_config( +cases() -> + [ #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE clientid = ${clientid} AND username = ${username}" - >> - } - ), + name => base_publish, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'b', 'allow', 'subscribe')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + client_info => #{username => <<"username">>}, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"a">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"b">>} + ] + }, + #{ + name => rule_by_clientid_cn_dn_peerhost, + setup => [ + "CREATE TABLE acl(clientid VARCHAR(255), cn VARCHAR(255), dn VARCHAR(255)," + " peerhost VARCHAR(255), topic VARCHAR(255)," + " permission VARCHAR(255), action VARCHAR(255))", - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). + "INSERT INTO acl(clientid, cn, dn, peerhost, topic, permission, action)" + " VALUES('clientid', 'cn', 'dn', '127.0.0.1', 'a', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic FROM acl WHERE" + " clientid = ${clientid} AND cn = ${cert_common_name}" + " AND dn = ${cert_subject} AND peerhost = ${peerhost}", + client_info => #{ + clientid => <<"clientid">>, + cn => <<"cn">>, + dn => <<"dn">>, + peerhost => {127, 0, 0, 1} + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>} + ] + }, + #{ + name => topics_literal_wildcard_variable, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/${username}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/${clientid}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 'eq t/${username}', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't/#', 'allow', 'publish')", + + "INSERT INTO acl(username, topic, permission, action) " + "VALUES('username', 't1/+', 'allow', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/${username}">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/1/2">>}, + {allow, ?AUTHZ_PUBLISH, <<"t1/1">>}, + {deny, ?AUTHZ_PUBLISH, <<"t1/1/2">>}, + {deny, ?AUTHZ_PUBLISH, <<"abc">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"t/username">>} + ] + }, + #{ + name => qos_retain_in_query_result, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255)," + "qos_s VARCHAR(255), retain_s VARCHAR(255))", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't1', 'allow', 'publish', '1', 'true')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't2', 'allow', 'publish', '2', 'false')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't3', 'allow', 'publish', '0,1,2', 'all')", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't4', 'allow', 'subscribe', '1', null)", + + "INSERT INTO acl(username, topic, permission, action, qos_s, retain_s)" + " VALUES('username', 't5', 'allow', 'subscribe', '0,1,2', null)" + ], + query => + "SELECT permission, action, topic, qos_s as qos, retain_s as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>}, + + {allow, ?AUTHZ_PUBLISH(2, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(2, true), <<"t2">>}, + + {allow, ?AUTHZ_PUBLISH(1, true), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(2, false), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(2, true), <<"t3">>}, + {allow, ?AUTHZ_PUBLISH(0, false), <<"t3">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"t4">>}, + {deny, ?AUTHZ_SUBSCRIBE(2), <<"t4">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"t5">>}, + {allow, ?AUTHZ_SUBSCRIBE(2), <<"t5">>}, + {allow, ?AUTHZ_SUBSCRIBE(0), <<"t5">>} + ] + }, + #{ + name => qos_retain_in_query_result_as_integer, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255)," + "qos_i VARCHAR(255), retain_i VARCHAR(255))", + + "INSERT INTO acl(username, topic, permission, action, qos_i, retain_i)" + " VALUES('username', 't1', 'allow', 'publish', 1, 1)" + ], + query => + "SELECT permission, action, topic, qos_i as qos, retain_i as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"t1">>} + ] + }, + #{ + name => retain_in_query_result_as_boolean, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255)," + " action VARCHAR(255), retain_b BOOLEAN)", + + "INSERT INTO acl(username, topic, permission, action, retain_b)" + " VALUES('username', 't1', 'allow', 'publish', true)", + + "INSERT INTO acl(username, topic, permission, action, retain_b)" + " VALUES('username', 't2', 'allow', 'publish', false)" + ], + query => + "SELECT permission, action, topic, retain_b as retain" + " FROM acl WHERE username = ${username}", + client_info => #{ + username => <<"username">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"t1">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"t1">>}, + {allow, ?AUTHZ_PUBLISH(1, false), <<"t2">>}, + {deny, ?AUTHZ_PUBLISH(1, true), <<"t2">>} + ] + }, + #{ + name => nonbin_values_in_client_info, + setup => [ + "CREATE TABLE acl(who VARCHAR(255), topic VARCHAR(255), permission VARCHAR(255)," + " action VARCHAR(255))", + + "INSERT INTO acl(who, topic, permission, action)" + " VALUES('username', 't/${username}', 'allow', 'publish')", + + "INSERT INTO acl(who, topic, permission, action)" + " VALUES('clientid', 't/${clientid}', 'allow', 'publish')" + ], + query => + "SELECT permission, action, topic" + " FROM acl WHERE who = ${username} OR who = ${clientid}", + client_info => #{ + %% string, not a binary + username => "username", + %% atom, not a binary + clientid => clientid + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {deny, ?AUTHZ_PUBLISH, <<"t/foo">>} + ] + }, + #{ + name => array_null_qos, + features => [rich_actions], + setup => [ + "CREATE TABLE acl(qos INTEGER[], " + " topic VARCHAR(255), permission VARCHAR(255), action VARCHAR(255))", + + "INSERT INTO acl(qos, topic, permission, action)" + " VALUES('{1,2}', 'tp', 'allow', 'publish')", + + "INSERT INTO acl(qos, topic, permission, action)" + " VALUES(NULL, 'ts', 'allow', 'subscribe')" + ], + query => + "SELECT permission, action, topic, qos FROM acl", + checks => [ + {allow, ?AUTHZ_PUBLISH(1, false), <<"tp">>}, + {allow, ?AUTHZ_PUBLISH(2, false), <<"tp">>}, + {deny, ?AUTHZ_PUBLISH(3, false), <<"tp">>}, + + {allow, ?AUTHZ_SUBSCRIBE(1), <<"ts">>}, + {allow, ?AUTHZ_SUBSCRIBE(2), <<"ts">>} + ] + }, + #{ + name => strip_double_quote, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'allow', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = \"${username}\"", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_query, + setup => [], + query => "SELECT permission, action, topic FROM acl WHER", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => pgsql_error, + setup => [], + query => + "SELECT permission, action, topic FROM table_not_exists WHERE username = ${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"t">>} + ] + } + %% TODO: add case for unknown variables after fixing EMQX-10400 + ]. %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ +setup_source_data(#{setup := Queries}) -> + lists:foreach( + fun(Query) -> + _ = q(Query) + end, + Queries + ). + +setup_authz_source(#{query := Query}) -> + setup_config( + #{ + <<"query">> => Query + } + ). + raw_pgsql_authz_config() -> #{ <<"enable">> => <<"true">>, @@ -331,61 +397,10 @@ q(Sql) -> {query, Sql} ). -insert(Sql, Params) -> - {ok, _} = emqx_resource:simple_sync_query( - ?PGSQL_RESOURCE, - {query, Sql, Params} - ), - ok. - -init_table() -> - ok = drop_table(), - {ok, _, _} = q( - "CREATE TABLE acl(\n" - " username VARCHAR(255),\n" - " clientid VARCHAR(255),\n" - " peerhost VARCHAR(255),\n" - " cn VARCHAR(255),\n" - " dn VARCHAR(255),\n" - " topic VARCHAR(255),\n" - " permission VARCHAR(255),\n" - " action VARCHAR(255))" - ), - ok. - drop_table() -> {ok, _, _} = q("DROP TABLE IF EXISTS acl"), ok. -setup_client_samples(ClientInfo, Samples) -> - #{username := Username} = ClientInfo, - ok = init_table(), - ok = lists:foreach( - fun(#{topics := Topics, permission := Permission, action := Action}) -> - lists:foreach( - fun(Topic) -> - insert( - << - "INSERT INTO acl(username, topic, permission, action)" - "VALUES($1, $2, $3, $4)" - >>, - [Username, Topic, Permission, Action] - ) - end, - Topics - ) - end, - Samples - ), - setup_config( - #{ - <<"query">> => << - "SELECT permission, action, topic " - "FROM acl WHERE username = ${username}" - >> - } - ). - setup_config(SpecialParams) -> emqx_authz_test_lib:setup_config( raw_pgsql_authz_config(), @@ -403,6 +418,15 @@ pgsql_config() -> ssl => #{enable => false} }. +create_pgsql_resource() -> + emqx_resource:create_local( + ?PGSQL_RESOURCE, + ?RESOURCE_GROUP, + emqx_connector_pgsql, + pgsql_config(), + #{} + ). + start_apps(Apps) -> lists:foreach(fun application:ensure_all_started/1, Apps). diff --git a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl index 29a352970..28110a7a5 100644 --- a/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_redis_SUITE.erl @@ -28,10 +28,10 @@ -define(REDIS_RESOURCE, <<"emqx_authz_redis_SUITE">>). all() -> - emqx_common_test_helpers:all(?MODULE). + emqx_authz_test_lib:all_with_table_case(?MODULE, t_run_case, cases()). groups() -> - []. + emqx_authz_test_lib:table_groups(t_run_case, cases()). init_per_suite(Config) -> ok = stop_apps([emqx_resource]), @@ -42,13 +42,7 @@ init_per_suite(Config) -> fun set_special_configs/1 ), ok = start_apps([emqx_resource]), - {ok, _} = emqx_resource:create_local( - ?REDIS_RESOURCE, - ?RESOURCE_GROUP, - emqx_redis, - redis_config(), - #{} - ), + ok = create_redis_resource(), Config; false -> {skip, no_redis} @@ -60,9 +54,18 @@ end_per_suite(_Config) -> ok = stop_apps([emqx_resource]), ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]). +init_per_group(Group, Config) -> + [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config]. +end_per_group(_Group, _Config) -> + ok. + init_per_testcase(_TestCase, Config) -> ok = emqx_authz_test_lib:reset_authorizers(), Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + _ = cleanup_redis(), + ok. set_special_configs(emqx_authz) -> ok = emqx_authz_test_lib:reset_authorizers(); @@ -73,93 +76,11 @@ set_special_configs(_) -> %% Tests %%------------------------------------------------------------------------------ -t_topic_rules(_Config) -> - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ok = emqx_authz_test_lib:test_no_topic_rules(ClientInfo, fun setup_client_samples/2), - - ok = emqx_authz_test_lib:test_allow_topic_rules(ClientInfo, fun setup_client_samples/2). - -t_lookups(_Config) -> - ClientInfo = #{ - clientid => <<"client id">>, - cn => <<"cn">>, - dn => <<"dn">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - - ByClientid = #{ - <<"mqtt_user:client id">> => - #{<<"a">> => <<"all">>} - }, - - ok = setup_sample(ByClientid), - ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${clientid}">>}), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByPeerhost = #{ - <<"mqtt_user:127.0.0.1">> => - #{<<"a">> => <<"all">>} - }, - - ok = setup_sample(ByPeerhost), - ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${peerhost}">>}), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByCN = #{ - <<"mqtt_user:cn">> => - #{<<"a">> => <<"all">>} - }, - - ok = setup_sample(ByCN), - ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${cert_common_name}">>}), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ), - - ByDN = #{ - <<"mqtt_user:dn">> => - #{<<"a">> => <<"all">>} - }, - - ok = setup_sample(ByDN), - ok = setup_config(#{<<"cmd">> => <<"HGETALL mqtt_user:${cert_subject}">>}), - - ok = emqx_authz_test_lib:test_samples( - ClientInfo, - [ - {allow, subscribe, <<"a">>}, - {deny, subscribe, <<"b">>} - ] - ). +t_run_case(Config) -> + Case = ?config(test_case, Config), + ok = setup_source_data(Case), + ok = setup_authz_source(Case), + ok = emqx_authz_test_lib:run_checks(Case). %% should still succeed to create even if the config will not work, %% because it's not a part of the schema check @@ -181,7 +102,7 @@ t_create_with_config_values_wont_work(_Config) -> InvalidConfigs ). -%% creating without a require field should return error +%% creating without a required field should return error t_create_invalid_config(_Config) -> AuthzConfig = raw_redis_authz_config(), C = maps:without([<<"server">>], AuthzConfig), @@ -196,54 +117,211 @@ t_create_invalid_config(_Config) -> t_redis_error(_Config) -> ok = setup_config(#{<<"cmd">> => <<"INVALID COMMAND">>}), - ClientInfo = #{ - clientid => <<"clientid">>, - username => <<"username">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ClientInfo = emqx_authz_test_lib:base_client_info(), - deny = emqx_access_control:authorize(ClientInfo, subscribe, <<"a">>). + ?assertEqual( + deny, + emqx_access_control:authorize(ClientInfo, ?AUTHZ_SUBSCRIBE, <<"a">>) + ). + +%%------------------------------------------------------------------------------ +%% Cases +%%------------------------------------------------------------------------------ + +cases() -> + [ + #{ + name => base_publish, + setup => [ + [ + "HMSET", + "acl:username", + "a", + "publish", + "b", + "subscribe", + "d", + "all" + ] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"a">>}, + + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"b">>}, + + {allow, ?AUTHZ_PUBLISH, <<"d">>}, + {allow, ?AUTHZ_SUBSCRIBE, <<"d">>} + ] + }, + #{ + name => invalid_rule, + setup => [ + [ + "HMSET", + "acl:username", + "a", + "[]", + "b", + "{invalid:json}", + "c", + "pub", + "d", + emqx_utils_json:encode(#{qos => 1, retain => true}) + ] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>}, + {deny, ?AUTHZ_PUBLISH, <<"b">>}, + {deny, ?AUTHZ_PUBLISH, <<"c">>}, + {deny, ?AUTHZ_PUBLISH(1, true), <<"d">>} + ] + }, + #{ + name => rule_by_clientid_cn_dn_peerhost, + setup => [ + ["HMSET", "acl:clientid:cn:dn:127.0.0.1", "a", "publish"] + ], + cmd => "HGETALL acl:${clientid}:${cert_common_name}:${cert_subject}:${peerhost}", + client_info => #{ + cn => <<"cn">>, + dn => <<"dn">> + }, + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => topics_literal_wildcard_variable, + setup => [ + [ + "HMSET", + "acl:username", + "t/${username}", + "publish", + "t/${clientid}", + "publish", + "t1/#", + "publish", + "t2/+", + "publish", + "eq t3/${username}", + "publish" + ] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"t/username">>}, + {allow, ?AUTHZ_PUBLISH, <<"t/clientid">>}, + {allow, ?AUTHZ_PUBLISH, <<"t1/a/b">>}, + {allow, ?AUTHZ_PUBLISH, <<"t2/a">>}, + {allow, ?AUTHZ_PUBLISH, <<"t3/${username}">>}, + {deny, ?AUTHZ_PUBLISH, <<"t3/username">>} + ] + }, + #{ + name => qos_retain_in_query_result, + features => [rich_actions], + setup => [ + [ + "HMSET", + "acl:username", + "a", + emqx_utils_json:encode(#{action => <<"publish">>, qos => 1, retain => true}), + "b", + emqx_utils_json:encode(#{ + action => <<"publish">>, qos => <<"1">>, retain => <<"true">> + }), + "c", + emqx_utils_json:encode(#{action => <<"publish">>, qos => <<"1,2">>, retain => 1}), + "d", + emqx_utils_json:encode(#{ + action => <<"publish">>, qos => [1, 2], retain => <<"1">> + }), + "e", + emqx_utils_json:encode(#{ + action => <<"publish">>, qos => [1, 2], retain => <<"all">> + }), + "f", + emqx_utils_json:encode(#{action => <<"publish">>, qos => null, retain => null}) + ] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {allow, ?AUTHZ_PUBLISH(1, true), <<"a">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"a">>}, + + {allow, ?AUTHZ_PUBLISH(1, true), <<"b">>}, + {deny, ?AUTHZ_PUBLISH(1, false), <<"b">>}, + {deny, ?AUTHZ_PUBLISH(2, false), <<"b">>}, + + {allow, ?AUTHZ_PUBLISH(2, true), <<"c">>}, + {deny, ?AUTHZ_PUBLISH(2, false), <<"c">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"c">>}, + + {allow, ?AUTHZ_PUBLISH(2, true), <<"d">>}, + {deny, ?AUTHZ_PUBLISH(0, true), <<"d">>}, + + {allow, ?AUTHZ_PUBLISH(1, false), <<"e">>}, + {allow, ?AUTHZ_PUBLISH(1, true), <<"e">>}, + {deny, ?AUTHZ_PUBLISH(0, false), <<"e">>}, + + {allow, ?AUTHZ_PUBLISH, <<"f">>}, + {deny, ?AUTHZ_SUBSCRIBE, <<"f">>} + ] + }, + #{ + name => nonbin_values_in_client_info, + setup => [ + [ + "HMSET", + "acl:username:clientid", + "a", + "publish" + ] + ], + client_info => #{ + username => "username", + clientid => clientid + }, + cmd => "HGETALL acl:${username}:${clientid}", + checks => [ + {allow, ?AUTHZ_PUBLISH, <<"a">>} + ] + }, + #{ + name => invalid_query, + setup => [ + ["SET", "acl:username", 1] + ], + cmd => "HGETALL acl:${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] + } + ]. %%------------------------------------------------------------------------------ %% Helpers %%------------------------------------------------------------------------------ -setup_sample(AuthzData) -> - {ok, _} = q(["FLUSHDB"]), - ok = lists:foreach( - fun({Key, Values}) -> - lists:foreach( - fun({TopicFilter, Action}) -> - q(["HSET", Key, TopicFilter, Action]) - end, - maps:to_list(Values) - ) +setup_source_data(#{setup := Queries}) -> + lists:foreach( + fun(Query) -> + _ = q(Query) end, - maps:to_list(AuthzData) + Queries ). -setup_client_samples(ClientInfo, Samples) -> - #{username := Username} = ClientInfo, - Key = <<"mqtt_user:", Username/binary>>, - lists:foreach( - fun(Sample) -> - #{ - topics := Topics, - permission := <<"allow">>, - action := Action - } = Sample, - lists:foreach( - fun(Topic) -> - q(["HSET", Key, Topic, Action]) - end, - Topics - ) - end, - Samples - ), - setup_config(#{}). +setup_authz_source(#{cmd := Cmd}) -> + setup_config( + #{ + <<"cmd">> => Cmd + } + ). setup_config(SpecialParams) -> Config = maps:merge(raw_redis_authz_config(), SpecialParams), @@ -261,6 +339,9 @@ raw_redis_authz_config() -> <<"server">> => <> }. +cleanup_redis() -> + q([<<"FLUSHALL">>]). + q(Command) -> emqx_resource:simple_sync_query( ?REDIS_RESOURCE, @@ -283,3 +364,13 @@ start_apps(Apps) -> stop_apps(Apps) -> lists:foreach(fun application:stop/1, Apps). + +create_redis_resource() -> + {ok, _} = emqx_resource:create_local( + ?REDIS_RESOURCE, + ?RESOURCE_GROUP, + emqx_redis, + redis_config(), + #{} + ), + ok. diff --git a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl index fbfb84785..c73fe96ea 100644 --- a/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_rule_SUITE.erl @@ -18,24 +18,17 @@ -compile(nowarn_export_all). -compile(export_all). --include("emqx_authz.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("emqx/include/emqx_placeholder.hrl"). --define(SOURCE1, {deny, all}). --define(SOURCE2, {allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}). --define(SOURCE3, {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]}). --define(SOURCE4, {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]}). --define(SOURCE5, - {allow, - {'or', [ - {username, {re, "^test"}}, - {clientid, {re, "test?"}} - ]}, - publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} -). --define(SOURCE6, {allow, {username, "test"}, publish, ["t/foo${username}boo"]}). +-define(CLIENT_INFO_BASE, #{ + clientid => <<"test">>, + username => <<"test">>, + peerhost => {127, 0, 0, 1}, + zone => default, + listener => {tcp, default} +}). all() -> emqx_common_test_helpers:all(?MODULE). @@ -59,6 +52,12 @@ end_per_suite(_Config) -> emqx_common_test_helpers:stop_apps([emqx_authz, emqx_conf]), ok. +init_per_testcase(_TestCase, Config) -> + Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ok. + set_special_configs(emqx_authz) -> {ok, _} = emqx:update_config([authorization, cache, enable], false), {ok, _} = emqx:update_config([authorization, no_match], deny), @@ -68,11 +67,11 @@ set_special_configs(_App) -> ok. t_compile(_) -> - ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile(?SOURCE1)), + ?assertEqual({deny, all, all, [['#']]}, emqx_authz_rule:compile({deny, all})), ?assertEqual( {allow, {ipaddr, {{127, 0, 0, 1}, {127, 0, 0, 1}, 32}}, all, [{eq, ['#']}, {eq, ['+']}]}, - emqx_authz_rule:compile(?SOURCE2) + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ), ?assertEqual( @@ -82,14 +81,18 @@ t_compile(_) -> {{192, 168, 1, 0}, {192, 168, 1, 255}, 24} ]}, subscribe, [{pattern, [{var, [<<"clientid">>]}]}]}, - emqx_authz_rule:compile(?SOURCE3) + emqx_authz_rule:compile( + {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} + ) ), - ?assertMatch( + ?assertEqual( {allow, {'and', [{clientid, {eq, <<"test">>}}, {username, {eq, <<"test">>}}]}, publish, [ [<<"topic">>, <<"test">>] ]}, - emqx_authz_rule:compile(?SOURCE4) + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) ), ?assertMatch( @@ -101,240 +104,643 @@ t_compile(_) -> publish, [ {pattern, [{var, [<<"username">>]}]}, {pattern, [{var, [<<"clientid">>]}]} ]}, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ), ?assertEqual( {allow, {username, {eq, <<"test">>}}, publish, [ {pattern, [{str, <<"t/foo">>}, {var, [<<"username">>]}, {str, <<"boo">>}]} ]}, - emqx_authz_rule:compile(?SOURCE6) + emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, + #{action_type => publish, qos => [0, 1, 2], retain => all}, [[<<"topic">>, <<"test">>]]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, #{action_type => publish, qos => [1], retain => true}, + [ + [<<"topic">>, <<"test">>] + ]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, #{action_type => subscribe, qos => [1, 2]}, [ + [<<"topic">>, <<"test">>] + ]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}, {qos, 2}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, #{action_type => subscribe, qos => [1]}, [ + [<<"topic">>, <<"test">>] + ]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, #{action_type => all, qos => [2], retain => true}, [ + [<<"topic">>, <<"test">>] + ]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ), + ok. +t_compile_ce(_Config) -> + _ = emqx_authz:set_feature_available(rich_actions, false), + + ?assertThrow( + {invalid_authorization_action, _}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ), + + ?assertEqual( + {allow, {username, {eq, <<"test">>}}, all, [[<<"topic">>, <<"test">>]]}, + emqx_authz_rule:compile( + {allow, {username, "test"}, all, ["topic/test"]} + ) + ). + t_match(_) -> - ClientInfo1 = #{ - clientid => <<"test">>, - username => <<"test">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - ClientInfo2 = #{ - clientid => <<"test">>, - username => <<"test">>, - peerhost => {192, 168, 1, 10}, - zone => default, - listener => {tcp, default} - }, - ClientInfo3 = #{ - clientid => <<"test">>, - username => <<"fake">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, - ClientInfo4 = #{ - clientid => <<"fake">>, - username => <<"test">>, - peerhost => {127, 0, 0, 1}, - zone => default, - listener => {tcp, default} - }, + ?assertEqual( + {matched, deny}, + emqx_authz_rule:match( + client_info(), + #{action_type => subscribe, qos => 0}, + <<"#">>, + emqx_authz_rule:compile({deny, all}) + ) + ), ?assertEqual( {matched, deny}, emqx_authz_rule:match( - ClientInfo1, - subscribe, - <<"#">>, - emqx_authz_rule:compile(?SOURCE1) - ) - ), - ?assertEqual( - {matched, deny}, - emqx_authz_rule:match( - ClientInfo2, - subscribe, + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => subscribe, qos => 0}, <<"+">>, - emqx_authz_rule:compile(?SOURCE1) + emqx_authz_rule:compile({deny, all}) ) ), + ?assertEqual( {matched, deny}, emqx_authz_rule:match( - ClientInfo3, - subscribe, + client_info(#{username => <<"fake">>}), + #{action_type => subscribe, qos => 0}, <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE1) + emqx_authz_rule:compile({deny, all}) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo1, - subscribe, + client_info(), + #{action_type => subscribe, qos => 0}, <<"#">>, - emqx_authz_rule:compile(?SOURCE2) + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ) ), + ?assertEqual( nomatch, emqx_authz_rule:match( - ClientInfo1, - subscribe, + client_info(), + #{action_type => subscribe, qos => 0}, <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE2) + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ) ), + ?assertEqual( nomatch, emqx_authz_rule:match( - ClientInfo2, - subscribe, + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => subscribe, qos => 0}, <<"#">>, - emqx_authz_rule:compile(?SOURCE2) + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, [{eq, "#"}, {eq, "+"}]}) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo1, - subscribe, + client_info(), + #{action_type => subscribe, qos => 0}, <<"test">>, - emqx_authz_rule:compile(?SOURCE3) - ) - ), - ?assertEqual( - {matched, allow}, - emqx_authz_rule:match( - ClientInfo2, - subscribe, - <<"test">>, - emqx_authz_rule:compile(?SOURCE3) - ) - ), - ?assertEqual( - nomatch, - emqx_authz_rule:match( - ClientInfo2, - subscribe, - <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE3) + emqx_authz_rule:compile( + {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} + ) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo1, - publish, - <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE4) - ) - ), - ?assertEqual( - {matched, allow}, - emqx_authz_rule:match( - ClientInfo2, - publish, - <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE4) + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => subscribe, qos => 0}, + <<"test">>, + emqx_authz_rule:compile( + {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} + ) ) ), + ?assertEqual( nomatch, emqx_authz_rule:match( - ClientInfo3, - publish, + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => subscribe, qos => 0}, <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE4) - ) - ), - ?assertEqual( - nomatch, - emqx_authz_rule:match( - ClientInfo4, - publish, - <<"topic/test">>, - emqx_authz_rule:compile(?SOURCE4) + emqx_authz_rule:compile( + {allow, {ipaddrs, ["127.0.0.1", "192.168.1.0/24"]}, subscribe, [?PH_S_CLIENTID]} + ) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo1, - publish, - <<"test">>, - emqx_authz_rule:compile(?SOURCE5) + client_info(), + #{action_type => publish, qos => 0, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) ) ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo2, - publish, - <<"test">>, - emqx_authz_rule:compile(?SOURCE5) + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => publish, qos => 0, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) ) ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{username => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {'and', [{client, "test"}, {user, "test"}]}, publish, ["topic/test"]} + ) + ) + ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo3, - publish, + client_info(), + #{action_type => publish, qos => 0, retain => false}, <<"test">>, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ) ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo3, - publish, + client_info(#{peerhost => {192, 168, 1, 10}}), + #{action_type => publish, qos => 0, retain => false}, + <<"test">>, + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{username => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, + <<"test">>, + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{username => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, <<"fake">>, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ) ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo4, - publish, + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, <<"test">>, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ) ), + ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo4, - publish, + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, <<"fake">>, - emqx_authz_rule:compile(?SOURCE5) + emqx_authz_rule:compile( + {allow, + {'or', [ + {username, {re, "^test"}}, + {clientid, {re, "test?"}} + ]}, + publish, [?PH_S_USERNAME, ?PH_S_CLIENTID]} + ) ) ), ?assertEqual( nomatch, emqx_authz_rule:match( - ClientInfo1, - publish, + client_info(), + #{action_type => publish, qos => 0, retain => false}, <<"t/foo${username}boo">>, - emqx_authz_rule:compile(?SOURCE6) + emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ) ), ?assertEqual( {matched, allow}, emqx_authz_rule:match( - ClientInfo4, - publish, + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => false}, <<"t/footestboo">>, - emqx_authz_rule:compile(?SOURCE6) + emqx_authz_rule:compile({allow, {username, "test"}, publish, ["t/foo${username}boo"]}) ) ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{retain, all}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 0, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => false}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{qos, 1}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 0}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, []}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, []}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 1}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 0}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 0}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 2, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ) + ), + + ?assertEqual( + {matched, allow}, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 2, retain => true}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, all, publish, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, all, publish, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{username => undefined, peerhost => undefined}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, {username, "user"}, all, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{username => undefined, peerhost => undefined}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, {ipaddr, "127.0.0.1"}, all, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{username => undefined, peerhost => undefined}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, {ipaddrs, []}, all, ["#"]}) + ) + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:match( + client_info(#{clientid => <<"fake">>}), + #{action_type => subscribe, qos => 2}, + <<"topic/test">>, + emqx_authz_rule:compile({allow, {clientid, {re, "^test"}}, all, ["#"]}) + ) + ), + ok. + +t_invalid_rule(_) -> + ?assertThrow( + {invalid_authorization_permission, _}, + emqx_authz_rule:compile({allawww, all, all, ["topic/test"]}) + ), + + ?assertThrow( + {invalid_authorization_rule, _}, + emqx_authz_rule:compile(ooops) + ), + + ?assertThrow( + {invalid_authorization_qos, _}, + emqx_authz_rule:compile({allow, {username, "test"}, {publish, [{qos, 3}]}, ["topic/test"]}) + ), + + ?assertThrow( + {invalid_authorization_retain, _}, + emqx_authz_rule:compile( + {allow, {username, "test"}, {publish, [{retain, 'FALSE'}]}, ["topic/test"]} + ) + ), + + ?assertThrow( + {invalid_authorization_action, _}, + emqx_authz_rule:compile({allow, all, unsubscribe, ["topic/test"]}) + ), + + ?assertThrow( + {invalid_who, _}, + emqx_authz_rule:compile({allow, who, all, ["topic/test"]}) + ). + +t_matches(_) -> + ?assertEqual( + {matched, allow}, + emqx_authz_rule:matches( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 2, retain => true}, + <<"topic/test">>, + [ + emqx_authz_rule:compile( + {allow, {username, "test"}, {subscribe, [{qos, 1}]}, ["topic/test"]} + ), + emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ) + ] + ) + ), + + Rule = emqx_authz_rule:compile( + {allow, {username, "test"}, {all, [{qos, 2}, {retain, true}]}, ["topic/test"]} + ), + + ?assertEqual( + nomatch, + emqx_authz_rule:matches( + client_info(#{clientid => <<"fake">>}), + #{action_type => publish, qos => 1, retain => true}, + <<"topic/test">>, + [Rule, Rule, Rule] + ) + ). + +%%-------------------------------------------------------------------- +%% Internal functions +%%-------------------------------------------------------------------- + +client_info() -> + ?CLIENT_INFO_BASE. + +client_info(Overrides) -> + maps:merge(?CLIENT_INFO_BASE, Overrides). diff --git a/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl new file mode 100644 index 000000000..8b097c3fd --- /dev/null +++ b/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl @@ -0,0 +1,274 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_authz_rule_raw_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +init_per_testcase(_TestCase, Config) -> + Config. +end_per_testcase(_TestCase, _Config) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ok. + +t_parse_ok(_Config) -> + lists:foreach( + fun({Expected, RuleRaw}) -> + _ = emqx_authz:set_feature_available(rich_actions, true), + ?assertEqual({ok, Expected}, emqx_authz_rule_raw:parse_rule(RuleRaw)), + _ = emqx_authz:set_feature_available(rich_actions, false), + ?assertEqual({ok, simple_rule(Expected)}, emqx_authz_rule_raw:parse_rule(RuleRaw)) + end, + ok_cases() + ). + +t_parse_error(_Config) -> + emqx_authz:set_feature_available(rich_actions, true), + lists:foreach( + fun(RuleRaw) -> + ?assertMatch( + {error, _}, + emqx_authz_rule_raw:parse_rule(RuleRaw) + ) + end, + error_cases() ++ error_rich_action_cases() + ), + + %% without rich actions some fields are not parsed, so they are not errors when invalid + _ = emqx_authz:set_feature_available(rich_actions, false), + lists:foreach( + fun(RuleRaw) -> + ?assertMatch( + {error, _}, + emqx_authz_rule_raw:parse_rule(RuleRaw) + ) + end, + error_cases() + ), + lists:foreach( + fun(RuleRaw) -> + ?assertMatch( + {ok, _}, + emqx_authz_rule_raw:parse_rule(RuleRaw) + ) + end, + error_rich_action_cases() + ). + +t_format(_Config) -> + ?assertEqual( + #{ + action => subscribe, + permission => allow, + qos => [1, 2], + retain => true, + topic => [<<"a/b/c">>] + }, + emqx_authz_rule_raw:format_rule( + {allow, {subscribe, [{qos, [1, 2]}, {retain, true}]}, [<<"a/b/c">>]} + ) + ), + ?assertEqual( + #{ + action => publish, + permission => allow, + topic => [<<"a/b/c">>] + }, + emqx_authz_rule_raw:format_rule( + {allow, publish, [<<"a/b/c">>]} + ) + ). + +t_format_no_rich_action(_Config) -> + _ = emqx_authz:set_feature_available(rich_actions, false), + + Rule = {allow, {subscribe, [{qos, [1, 2]}, {retain, true}]}, [<<"a/b/c">>]}, + + ?assertEqual( + #{action => subscribe, permission => allow, topic => [<<"a/b/c">>]}, + emqx_authz_rule_raw:format_rule(Rule) + ). + +%%-------------------------------------------------------------------- +%% Cases +%%-------------------------------------------------------------------- + +ok_cases() -> + [ + { + {allow, {publish, [{qos, [0, 1, 2]}, {retain, all}]}, [<<"a/b/c">>]}, + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => <<"a/b/c">>, + <<"action">> => <<"publish">> + } + }, + { + {deny, {subscribe, [{qos, [1, 2]}]}, [{eq, <<"a/b/c">>}]}, + #{ + <<"permission">> => <<"deny">>, + <<"topic">> => <<"eq a/b/c">>, + <<"action">> => <<"subscribe">>, + <<"retain">> => <<"true">>, + <<"qos">> => <<"1,2">> + } + }, + { + {allow, {publish, [{qos, [0, 1, 2]}, {retain, all}]}, [<<"a">>, <<"b">>]}, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [<<"a">>, <<"b">>], + <<"action">> => <<"publish">> + } + }, + { + {allow, {all, [{qos, [0, 1, 2]}, {retain, all}]}, []}, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"all">> + } + }, + %% Retain + { + expected_rule_with_qos_retain([0, 1, 2], true), + rule_with_raw_qos_retain(#{<<"retain">> => <<"true">>}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], true), + rule_with_raw_qos_retain(#{<<"retain">> => true}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], false), + rule_with_raw_qos_retain(#{<<"retain">> => false}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], false), + rule_with_raw_qos_retain(#{<<"retain">> => <<"false">>}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"retain">> => <<"all">>}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"retain">> => undefined}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"retain">> => null}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{}) + }, + %% Qos + { + expected_rule_with_qos_retain([2], all), + rule_with_raw_qos_retain(#{<<"qos">> => <<"2">>}) + }, + { + expected_rule_with_qos_retain([2], all), + rule_with_raw_qos_retain(#{<<"qos">> => [<<"2">>]}) + }, + { + expected_rule_with_qos_retain([1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => <<"1,2">>}) + }, + { + expected_rule_with_qos_retain([1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => [<<"1">>, <<"2">>]}) + }, + { + expected_rule_with_qos_retain([1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => [1, 2]}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => undefined}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], all), + rule_with_raw_qos_retain(#{<<"qos">> => null}) + } + ]. + +error_cases() -> + [ + #{ + <<"permission">> => <<"allo">>, + <<"topic">> => <<"a/b/c">>, + <<"action">> => <<"publish">> + }, + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => <<"a/b/c">>, + <<"action">> => <<"publis">> + }, + #{ + <<"permission">> => <<"allow">>, + <<"topic">> => #{}, + <<"action">> => <<"publish">> + }, + #{ + <<"permission">> => <<"allow">>, + <<"action">> => <<"publish">> + } + ]. + +error_rich_action_cases() -> + [ + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">>, + <<"qos">> => 3 + }, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">>, + <<"qos">> => <<"three">> + }, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">>, + <<"retain">> => 3 + } + ]. + +expected_rule_with_qos_retain(QoS, Retain) -> + {allow, {publish, [{qos, QoS}, {retain, Retain}]}, []}. + +rule_with_raw_qos_retain(Overrides) -> + maps:merge(base_raw_rule(), Overrides). + +base_raw_rule() -> + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">> + }. + +simple_rule({Pemission, {Action, _Opts}, Topics}) -> + {Pemission, Action, Topics}. diff --git a/apps/emqx_authz/test/emqx_authz_test_lib.erl b/apps/emqx_authz/test/emqx_authz_test_lib.erl index 03d687851..ea62c5b71 100644 --- a/apps/emqx_authz/test/emqx_authz_test_lib.erl +++ b/apps/emqx_authz/test/emqx_authz_test_lib.erl @@ -22,8 +22,6 @@ -compile(nowarn_export_all). -compile(export_all). --define(DEFAULT_CHECK_AVAIL_TIMEOUT, 1000). - reset_authorizers() -> reset_authorizers(deny, false, []). @@ -53,216 +51,68 @@ setup_config(BaseConfig, SpecialParams) -> {error, Reason} -> {error, Reason} end. -test_samples(ClientInfo, Samples) -> +%%-------------------------------------------------------------------- +%% Table-based test helpers +%%-------------------------------------------------------------------- + +all_with_table_case(Mod, TableCase, Cases) -> + (emqx_common_test_helpers:all(Mod) -- [TableCase]) ++ + [{group, Name} || Name <- case_names(Cases)]. + +table_groups(TableCase, Cases) -> + [{Name, [], [TableCase]} || Name <- case_names(Cases)]. + +case_names(Cases) -> + lists:map(fun(Case) -> maps:get(name, Case) end, Cases). + +get_case(Name, Cases) -> + [Case] = [C || C <- Cases, maps:get(name, C) =:= Name], + Case. + +setup_default_permission(Case) -> + DefaultPermission = maps:get(default_permission, Case, deny), + emqx_authz_test_lib:reset_authorizers(DefaultPermission, false). + +base_client_info() -> + #{ + clientid => <<"clientid">>, + username => <<"username">>, + peerhost => {127, 0, 0, 1}, + zone => default, + listener => {tcp, default} + }. + +client_info(Overrides) -> + maps:merge(base_client_info(), Overrides). + +enable_features(Case) -> + Features = maps:get(features, Case, []), lists:foreach( - fun({Expected, Action, Topic}) -> - ct:pal( - "client_info: ~p, action: ~p, topic: ~p, expected: ~p", - [ClientInfo, Action, Topic, Expected] - ), - ?assertEqual( - Expected, - emqx_access_control:authorize( - ClientInfo, - Action, - Topic - ) - ) + fun(Feature) -> + Enable = lists:member(Feature, Features), + emqx_authz:set_feature_available(Feature, Enable) end, - Samples + ?AUTHZ_FEATURES ). -test_no_topic_rules(ClientInfo, SetupSamples) -> - %% No rules - - ok = reset_authorizers(deny, false), - ok = SetupSamples(ClientInfo, []), - - ok = test_samples( - ClientInfo, - [ - {deny, subscribe, <<"#">>}, - {deny, subscribe, <<"subs">>}, - {deny, publish, <<"pub">>} - ] +run_checks(#{checks := Checks} = Case) -> + _ = setup_default_permission(Case), + _ = enable_features(Case), + ClientInfoOverrides = maps:get(client_info, Case, #{}), + ClientInfo = client_info(ClientInfoOverrides), + lists:foreach( + fun(Check) -> + run_check(ClientInfo, Check) + end, + Checks ). -test_allow_topic_rules(ClientInfo, SetupSamples) -> - Samples = [ - #{ - topics => [ - <<"eq testpub1/${username}">>, - <<"testpub2/${clientid}">>, - <<"testpub3/#">> - ], - permission => <<"allow">>, - action => <<"publish">> - }, - #{ - topics => [ - <<"eq testsub1/${username}">>, - <<"testsub2/${clientid}">>, - <<"testsub3/#">> - ], - permission => <<"allow">>, - action => <<"subscribe">> - }, - - #{ - topics => [ - <<"eq testall1/${username}">>, - <<"testall2/${clientid}">>, - <<"testall3/#">> - ], - permission => <<"allow">>, - action => <<"all">> - } - ], - - ok = reset_authorizers(deny, false), - ok = SetupSamples(ClientInfo, Samples), - - ok = test_samples( - ClientInfo, - [ - %% Publish rules - - {deny, publish, <<"testpub1/username">>}, - {allow, publish, <<"testpub1/${username}">>}, - {allow, publish, <<"testpub2/clientid">>}, - {allow, publish, <<"testpub3/foobar">>}, - - {deny, publish, <<"testpub2/username">>}, - {deny, publish, <<"testpub1/clientid">>}, - - {deny, subscribe, <<"testpub1/username">>}, - {deny, subscribe, <<"testpub2/clientid">>}, - {deny, subscribe, <<"testpub3/foobar">>}, - - %% Subscribe rules - - {deny, subscribe, <<"testsub1/username">>}, - {allow, subscribe, <<"testsub1/${username}">>}, - {allow, subscribe, <<"testsub2/clientid">>}, - {allow, subscribe, <<"testsub3/foobar">>}, - {allow, subscribe, <<"testsub3/+/foobar">>}, - {allow, subscribe, <<"testsub3/#">>}, - - {deny, subscribe, <<"testsub2/username">>}, - {deny, subscribe, <<"testsub1/clientid">>}, - {deny, subscribe, <<"testsub4/foobar">>}, - {deny, publish, <<"testsub1/username">>}, - {deny, publish, <<"testsub2/clientid">>}, - {deny, publish, <<"testsub3/foobar">>}, - - %% All rules - - {deny, subscribe, <<"testall1/username">>}, - {allow, subscribe, <<"testall1/${username}">>}, - {allow, subscribe, <<"testall2/clientid">>}, - {allow, subscribe, <<"testall3/foobar">>}, - {allow, subscribe, <<"testall3/+/foobar">>}, - {allow, subscribe, <<"testall3/#">>}, - {deny, publish, <<"testall1/username">>}, - {allow, publish, <<"testall1/${username}">>}, - {allow, publish, <<"testall2/clientid">>}, - {allow, publish, <<"testall3/foobar">>}, - - {deny, subscribe, <<"testall2/username">>}, - {deny, subscribe, <<"testall1/clientid">>}, - {deny, subscribe, <<"testall4/foobar">>}, - {deny, publish, <<"testall2/username">>}, - {deny, publish, <<"testall1/clientid">>}, - {deny, publish, <<"testall4/foobar">>} - ] - ). - -test_deny_topic_rules(ClientInfo, SetupSamples) -> - Samples = [ - #{ - topics => [ - <<"eq testpub1/${username}">>, - <<"testpub2/${clientid}">>, - <<"testpub3/#">> - ], - permission => <<"deny">>, - action => <<"publish">> - }, - #{ - topics => [ - <<"eq testsub1/${username}">>, - <<"testsub2/${clientid}">>, - <<"testsub3/#">> - ], - permission => <<"deny">>, - action => <<"subscribe">> - }, - - #{ - topics => [ - <<"eq testall1/${username}">>, - <<"testall2/${clientid}">>, - <<"testall3/#">> - ], - permission => <<"deny">>, - action => <<"all">> - } - ], - - ok = reset_authorizers(allow, false), - ok = SetupSamples(ClientInfo, Samples), - - ok = test_samples( - ClientInfo, - [ - %% Publish rules - - {allow, publish, <<"testpub1/username">>}, - {deny, publish, <<"testpub1/${username}">>}, - {deny, publish, <<"testpub2/clientid">>}, - {deny, publish, <<"testpub3/foobar">>}, - - {allow, publish, <<"testpub2/username">>}, - {allow, publish, <<"testpub1/clientid">>}, - - {allow, subscribe, <<"testpub1/username">>}, - {allow, subscribe, <<"testpub2/clientid">>}, - {allow, subscribe, <<"testpub3/foobar">>}, - - %% Subscribe rules - - {allow, subscribe, <<"testsub1/username">>}, - {deny, subscribe, <<"testsub1/${username}">>}, - {deny, subscribe, <<"testsub2/clientid">>}, - {deny, subscribe, <<"testsub3/foobar">>}, - {deny, subscribe, <<"testsub3/+/foobar">>}, - {deny, subscribe, <<"testsub3/#">>}, - - {allow, subscribe, <<"testsub2/username">>}, - {allow, subscribe, <<"testsub1/clientid">>}, - {allow, subscribe, <<"testsub4/foobar">>}, - {allow, publish, <<"testsub1/username">>}, - {allow, publish, <<"testsub2/clientid">>}, - {allow, publish, <<"testsub3/foobar">>}, - - %% All rules - - {allow, subscribe, <<"testall1/username">>}, - {deny, subscribe, <<"testall1/${username}">>}, - {deny, subscribe, <<"testall2/clientid">>}, - {deny, subscribe, <<"testall3/foobar">>}, - {deny, subscribe, <<"testall3/+/foobar">>}, - {deny, subscribe, <<"testall3/#">>}, - {allow, publish, <<"testall1/username">>}, - {deny, publish, <<"testall1/${username}">>}, - {deny, publish, <<"testall2/clientid">>}, - {deny, publish, <<"testall3/foobar">>}, - - {allow, subscribe, <<"testall2/username">>}, - {allow, subscribe, <<"testall1/clientid">>}, - {allow, subscribe, <<"testall4/foobar">>}, - {allow, publish, <<"testall2/username">>}, - {allow, publish, <<"testall1/clientid">>}, - {allow, publish, <<"testall4/foobar">>} - ] +run_check(ClientInfo, {ExpectedPermission, Action, Topic}) -> + ?assertEqual( + ExpectedPermission, + emqx_access_control:authorize( + ClientInfo, + Action, + Topic + ) ). diff --git a/apps/emqx_exhook/src/emqx_exhook.app.src b/apps/emqx_exhook/src/emqx_exhook.app.src index 92a70cf37..8a57249e9 100644 --- a/apps/emqx_exhook/src/emqx_exhook.app.src +++ b/apps/emqx_exhook/src/emqx_exhook.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_exhook, [ {description, "EMQX Extension for Hook"}, - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, []}, {registered, []}, {mod, {emqx_exhook_app, []}}, diff --git a/apps/emqx_exhook/src/emqx_exhook_handler.erl b/apps/emqx_exhook/src/emqx_exhook_handler.erl index 8720f65ae..b4358969d 100644 --- a/apps/emqx_exhook/src/emqx_exhook_handler.erl +++ b/apps/emqx_exhook/src/emqx_exhook_handler.erl @@ -16,9 +16,9 @@ -module(emqx_exhook_handler). --include("emqx_exhook.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -export([ on_client_connect/2, @@ -132,12 +132,13 @@ on_client_authenticate(ClientInfo, AuthResult) -> {ok, AuthResult} end. -on_client_authorize(ClientInfo, PubSub, Topic, Result) -> +on_client_authorize(ClientInfo, Action, Topic, Result) -> Bool = maps:get(result, Result, deny) == allow, + %% TODO: Support full action in major release Type = - case PubSub of - publish -> 'PUBLISH'; - subscribe -> 'SUBSCRIBE' + case Action of + ?authz_action(publish) -> 'PUBLISH'; + ?authz_action(subscribe) -> 'SUBSCRIBE' end, Req = #{ clientinfo => clientinfo(ClientInfo), diff --git a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl index bd756620d..3da73c11a 100644 --- a/apps/emqx_exhook/test/emqx_exhook_SUITE.erl +++ b/apps/emqx_exhook/test/emqx_exhook_SUITE.erl @@ -19,7 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("emqx_exhook.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). @@ -126,7 +126,7 @@ t_access_failed_if_no_server_running(Config) -> allow, emqx_access_control:authorize( ClientInfo#{username => <<"gooduser">>}, - publish, + ?AUTHZ_PUBLISH, <<"acl/1">> ) ), @@ -135,7 +135,7 @@ t_access_failed_if_no_server_running(Config) -> deny, emqx_access_control:authorize( ClientInfo#{username => <<"baduser">>}, - publish, + ?AUTHZ_PUBLISH, <<"acl/2">> ) ), @@ -148,7 +148,7 @@ t_access_failed_if_no_server_running(Config) -> ?assertMatch( {stop, #{result := deny, from := exhook}}, - emqx_exhook_handler:on_client_authorize(ClientInfo, publish, <<"t/1">>, #{ + emqx_exhook_handler:on_client_authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t/1">>, #{ result => allow, from => exhook }) ), diff --git a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl index 075cc736c..34d7a4342 100644 --- a/apps/emqx_exhook/test/props/prop_exhook_hooks.erl +++ b/apps/emqx_exhook/test/props/prop_exhook_hooks.erl @@ -18,6 +18,7 @@ -include_lib("proper/include/proper.hrl"). -include_lib("eunit/include/eunit.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -import( emqx_proper_types, @@ -29,7 +30,8 @@ connack_return_code/0, topictab/0, topic/0, - subopts/0 + subopts/0, + pubsub/0 ] ). @@ -138,7 +140,7 @@ prop_client_authorize() -> {ClientInfo0, PubSub, Topic, Result, Meta}, { clientinfo(), - oneof([publish, subscribe]), + pubsub(), topic(), oneof([MkResult(allow), MkResult(deny)]), request_meta() @@ -554,8 +556,8 @@ authresult_to_bool(AuthResult) -> aclresult_to_bool(#{result := Result}) -> Result == allow. -pubsub_to_enum(publish) -> 'PUBLISH'; -pubsub_to_enum(subscribe) -> 'SUBSCRIBE'. +pubsub_to_enum(?authz_action(publish)) -> 'PUBLISH'; +pubsub_to_enum(?authz_action(subscribe)) -> 'SUBSCRIBE'. from_conninfo(ConnInfo) -> #{ diff --git a/apps/emqx_gateway/src/emqx_gateway_ctx.erl b/apps/emqx_gateway/src/emqx_gateway_ctx.erl index 32e5fcf96..11ad55d3e 100644 --- a/apps/emqx_gateway/src/emqx_gateway_ctx.erl +++ b/apps/emqx_gateway/src/emqx_gateway_ctx.erl @@ -162,8 +162,8 @@ connection_closed(_Ctx = #{gwname := GwName}, ClientId) -> emqx_types:topic() ) -> allow | deny. -authorize(_Ctx, ClientInfo, PubSub, Topic) -> - emqx_access_control:authorize(ClientInfo, PubSub, Topic). +authorize(_Ctx, ClientInfo, Action, Topic) -> + emqx_access_control:authorize(ClientInfo, Action, Topic). metrics_inc(_Ctx = #{gwname := GwName}, Name) -> emqx_gateway_metrics:inc(GwName, Name). diff --git a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl index d95dc5bd2..0c0b7310d 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_channel.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_channel.erl @@ -47,9 +47,6 @@ -include("emqx_coap.hrl"). -include_lib("emqx/include/logger.hrl"). --include_lib("emqx/include/emqx_authentication.hrl"). - --define(AUTHN, ?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_ATOM). -record(channel, { %% Context @@ -166,8 +163,8 @@ init( conn_state = idle }. -validator(Type, Topic, Ctx, ClientInfo) -> - emqx_gateway_ctx:authorize(Ctx, ClientInfo, Type, Topic). +validator(Action, Topic, Ctx, ClientInfo) -> + emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic). -spec send_request(pid(), coap_message()) -> any(). send_request(Channel, Request) -> diff --git a/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl b/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl index da1f5e0ef..3070ea891 100644 --- a/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl +++ b/apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl @@ -18,6 +18,7 @@ -module(emqx_coap_pubsub_handler). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include("emqx_coap.hrl"). -export([handle_request/4]). @@ -50,14 +51,16 @@ handle_method(get, Topic, Msg, Ctx, CInfo) -> reply({error, bad_request}, <<"invalid observe value">>, Msg) end; handle_method(post, Topic, #coap_message{payload = Payload} = Msg, Ctx, CInfo) -> - case emqx_coap_channel:validator(publish, Topic, Ctx, CInfo) of + PublishOpts = get_publish_opts(Msg), + Qos = get_publish_qos(Msg, PublishOpts), + Action = ?AUTHZ_PUBLISH(Qos, get_publish_retain(PublishOpts)), + case emqx_coap_channel:validator(Action, Topic, Ctx, CInfo) of allow -> #{clientid := ClientId} = CInfo, MountTopic = mount(CInfo, Topic), - QOS = get_publish_qos(Msg), %% TODO: Append message metadata into headers - MQTTMsg = emqx_message:make(ClientId, QOS, MountTopic, Payload), - MQTTMsg2 = apply_publish_opts(Msg, MQTTMsg), + MQTTMsg = emqx_message:make(ClientId, Qos, MountTopic, Payload), + MQTTMsg2 = apply_publish_opts(PublishOpts, MQTTMsg), _ = emqx_broker:publish(MQTTMsg2), reply({ok, changed}, Msg); _ -> @@ -104,48 +107,70 @@ type_to_qos(coap, #coap_message{type = Type}) -> ?QOS_1 end. -get_publish_qos(Msg) -> - case emqx_coap_message:get_option(uri_query, Msg) of - #{<<"qos">> := QOS} -> - erlang:binary_to_integer(QOS); - _ -> - CfgType = emqx_conf:get([gateway, coap, publish_qos], ?QOS_0), - type_to_qos(CfgType, Msg) - end. - -apply_publish_opts(Msg, MQTTMsg) -> +get_publish_opts(Msg) -> case emqx_coap_message:get_option(uri_query, Msg) of undefined -> - MQTTMsg; + #{}; Qs -> maps:fold( fun (<<"retain">>, V, Acc) -> Val = V =:= <<"true">>, - emqx_message:set_flag(retain, Val, Acc); + Acc#{retain => Val}; (<<"expiry">>, V, Acc) -> Val = erlang:binary_to_integer(V), - Props = emqx_message:get_header(properties, Acc), - emqx_message:set_header( - properties, - Props#{'Message-Expiry-Interval' => Val}, - Acc - ); + Acc#{expiry_interval => Val}; + (<<"qos">>, V, Acc) -> + Val = erlang:binary_to_integer(V), + Acc#{qos => Val}; (_, _, Acc) -> Acc end, - MQTTMsg, + #{}, Qs ) end. +get_publish_qos(Msg, PublishOpts) -> + case PublishOpts of + #{qos := Qos} -> + Qos; + _ -> + CfgType = emqx_conf:get([gateway, coap, publish_qos], ?QOS_0), + type_to_qos(CfgType, Msg) + end. + +get_publish_retain(PublishOpts) -> + maps:get(retain, PublishOpts, false). + +apply_publish_opts(Opts, MQTTMsg) -> + maps:fold( + fun + (retain, Val, Acc) -> + emqx_message:set_flag(retain, Val, Acc); + (expiry, Val, Acc) -> + Props = emqx_message:get_header(properties, Acc), + emqx_message:set_header( + properties, + Props#{'Message-Expiry-Interval' => Val}, + Acc + ); + (_, _, Acc) -> + Acc + end, + MQTTMsg, + Opts + ). + subscribe(#coap_message{token = <<>>} = Msg, _, _, _) -> reply({error, bad_request}, <<"observe without token">>, Msg); subscribe(#coap_message{token = Token} = Msg, Topic, Ctx, CInfo) -> - case emqx_coap_channel:validator(subscribe, Topic, Ctx, CInfo) of + #{qos := Qos} = SubOpts = get_sub_opts(Msg), + Action = ?AUTHZ_SUBSCRIBE(Qos), + case emqx_coap_channel:validator(Action, Topic, Ctx, CInfo) of allow -> #{clientid := ClientId} = CInfo, - SubOpts = get_sub_opts(Msg), + MountTopic = mount(CInfo, Topic), emqx_broker:subscribe(MountTopic, ClientId, SubOpts), run_hooks(Ctx, 'session.subscribed', [CInfo, MountTopic, SubOpts]), diff --git a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl index 5de597920..80d3282c5 100644 --- a/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_gateway_exproto/src/emqx_exproto_channel.erl @@ -19,6 +19,7 @@ -include("emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). @@ -428,7 +429,8 @@ handle_call( clientinfo = ClientInfo } ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, TopicFilter) of + Action = ?AUTHZ_SUBSCRIBE(Qos), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, TopicFilter) of deny -> {reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel}; _ -> @@ -464,7 +466,8 @@ handle_call( } } ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + Action = ?AUTHZ_PUBLISH(Qos), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic) of deny -> {reply, {error, ?RESP_PERMISSION_DENY, <<"Authorization deny">>}, Channel}; _ -> diff --git a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl index bbd2d4377..e187b3fb7 100644 --- a/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl +++ b/apps/emqx_gateway_lwm2m/src/emqx_lwm2m_channel.erl @@ -17,7 +17,9 @@ -module(emqx_lwm2m_channel). -include("emqx_lwm2m.hrl"). +-include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx_gateway_coap/include/emqx_coap.hrl"). %% API @@ -644,7 +646,8 @@ with_context(Ctx, ClientInfo) -> end. with_context(publish, [Topic, Msg], Ctx, ClientInfo) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + Action = publish_action(Msg), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic) of allow -> _ = emqx_broker:publish(Msg), ok; @@ -660,7 +663,8 @@ with_context(subscribe, [Topic, Opts], Ctx, ClientInfo) -> clientid := ClientId, endpoint_name := EndpointName } = ClientInfo, - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, Topic) of + Action = subscribe_action(Opts), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic) of allow -> run_hooks(Ctx, 'session.subscribed', [ClientInfo, Topic, Opts]), ?SLOG(debug, #{ @@ -681,6 +685,14 @@ with_context(subscribe, [Topic, Opts], Ctx, ClientInfo) -> with_context(metrics, Name, Ctx, _ClientInfo) -> emqx_gateway_ctx:metrics_inc(Ctx, Name). +publish_action(#message{qos = QoS, flags = Flags}) -> + Retain = maps:get(retain, Flags, false), + ?AUTHZ_PUBLISH(QoS, Retain). + +subscribe_action(Opts) -> + QoS = maps:get(qos, Opts, 0), + ?AUTHZ_SUBSCRIBE(QoS). + %%-------------------------------------------------------------------- %% Call Chain %%-------------------------------------------------------------------- diff --git a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src index b43201e1a..5e79d4d49 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src +++ b/apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src @@ -1,6 +1,6 @@ {application, emqx_gateway_mqttsn, [ {description, "MQTT-SN Gateway"}, - {vsn, "0.1.2"}, + {vsn, "0.1.3"}, {registered, []}, {applications, [kernel, stdlib, emqx, emqx_gateway]}, {env, []}, diff --git a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl index 720c288d3..2443b149a 100644 --- a/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl +++ b/apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl @@ -22,6 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/types.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). @@ -1099,10 +1100,11 @@ convert_topic_id_to_name( end. check_pub_authz( - {TopicName, _Flags, _Data}, + {TopicName, #mqtt_sn_flags{qos = QoS, retain = Retain}, _Data}, #channel{ctx = Ctx, clientinfo = ClientInfo} ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, TopicName) of + Action = ?AUTHZ_PUBLISH(QoS, Retain), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, TopicName) of allow -> ok; deny -> {error, ?SN_RC2_NOT_AUTHORIZE} end. @@ -1251,10 +1253,11 @@ preproc_subs_type( {error, ?SN_RC_NOT_SUPPORTED}. check_subscribe_authz( - {_TopicId, TopicName, _QoS}, + {_TopicId, TopicName, QoS}, Channel = #channel{ctx = Ctx, clientinfo = ClientInfo} ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, TopicName) of + Action = ?AUTHZ_SUBSCRIBE(QoS), + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, TopicName) of allow -> {ok, Channel}; _ -> diff --git a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl index 3577c0a60..3ae928ba3 100644 --- a/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl +++ b/apps/emqx_gateway_stomp/src/emqx_stomp_channel.erl @@ -20,6 +20,7 @@ -include("emqx_stomp.hrl"). -include_lib("emqx/include/emqx.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx/include/logger.hrl"). -import(proplists, [get_value/2, get_value/3]). @@ -446,7 +447,10 @@ handle_in( } ) -> Topic = header(<<"destination">>, Headers), - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, publish, Topic) of + %% Flags and QoS are not supported in STOMP anyway, + %% no need to look into the frame + Action = ?AUTHZ_PUBLISH, + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, Topic) of deny -> ErrMsg = io_lib:format("Insufficient permissions for ~s", [Topic]), ErrorFrame = error_frame(receipt_id(Headers), ErrMsg), @@ -717,7 +721,9 @@ check_sub_acl( clientinfo = ClientInfo } ) -> - case emqx_gateway_ctx:authorize(Ctx, ClientInfo, subscribe, ParsedTopic) of + %% QoS is not supported in stomp + Action = ?AUTHZ_SUBSCRIBE, + case emqx_gateway_ctx:authorize(Ctx, ClientInfo, Action, ParsedTopic) of deny -> {error, acl_denied}; allow -> ok end. diff --git a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl index 6e3768431..47756cc4c 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl @@ -89,15 +89,15 @@ t_clients(_) -> AfterKickoutResponse2 = emqx_mgmt_api_test_util:request_api(get, Client2Path), ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse2), - %% get /clients/:clientid/authorization/cache should has no authz cache + %% get /clients/:clientid/authorization/cache should have no authz cache Client1AuthzCachePath = emqx_mgmt_api_test_util:api_path([ "clients", binary_to_list(ClientId1), "authorization", "cache" ]), - {ok, Client1AuthzCache} = emqx_mgmt_api_test_util:request_api(get, Client1AuthzCachePath), - ?assertEqual("[]", Client1AuthzCache), + {ok, Client1AuthzCache0} = emqx_mgmt_api_test_util:request_api(get, Client1AuthzCachePath), + ?assertEqual("[]", Client1AuthzCache0), %% post /clients/:clientid/subscribe SubscribeBody = #{topic => Topic, qos => Qos, nl => 1, rh => 1}, @@ -167,6 +167,35 @@ t_clients(_) -> AfterKickoutResponse1 = emqx_mgmt_api_test_util:request_api(get, Client1Path), ?assertEqual({error, {"HTTP/1.1", 404, "Not Found"}}, AfterKickoutResponse1). +t_authz_cache(_) -> + ClientId = <<"client_authz">>, + + {ok, C} = emqtt:start_link(#{clientid => ClientId}), + {ok, _} = emqtt:connect(C), + {ok, _, _} = emqtt:subscribe(C, <<"topic/1">>, 0), + + ClientAuthzCachePath = emqx_mgmt_api_test_util:api_path([ + "clients", + binary_to_list(ClientId), + "authorization", + "cache" + ]), + {ok, ClientAuthzCache} = emqx_mgmt_api_test_util:request_api(get, ClientAuthzCachePath), + ?assertMatch( + [ + #{ + <<"access">> := + #{<<"action_type">> := <<"subscribe">>, <<"qos">> := 1}, + <<"result">> := <<"allow">>, + <<"topic">> := <<"topic/1">>, + <<"updated_time">> := _ + } + ], + emqx_utils_json:decode(ClientAuthzCache, [return_maps]) + ), + + ok = emqtt:stop(C). + t_kickout_clients(_) -> process_flag(trap_exit, true), diff --git a/apps/emqx_rule_engine/src/emqx_rule_events.erl b/apps/emqx_rule_engine/src/emqx_rule_events.erl index 7f14f6d8b..3ff588f48 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_events.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_events.erl @@ -20,6 +20,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("emqx/include/emqx_hooks.hrl"). +-include_lib("emqx/include/emqx_access_control.hrl"). -include_lib("emqx_bridge/include/emqx_bridge_resource.hrl"). -export([ @@ -160,7 +161,10 @@ on_client_connack(ConnInfo, Reason, _, Conf) -> Conf ). -on_client_check_authz_complete(ClientInfo, PubSub, Topic, Result, AuthzSource, Conf) -> +%% TODO: support full action in major release +on_client_check_authz_complete( + ClientInfo, ?authz_action(PubSub), Topic, Result, AuthzSource, Conf +) -> apply_event( 'client.check_authz_complete', fun() -> diff --git a/changes/ee/feat-11132.en.md b/changes/ee/feat-11132.en.md new file mode 100644 index 000000000..6ebc7efe2 --- /dev/null +++ b/changes/ee/feat-11132.en.md @@ -0,0 +1,2 @@ +Add support for MQTT action authorization based on QoS level and Retain flag values. +Now, EMQX can check by ACL whether a client has permission to publish/subscribe using a specified QoS level and to use retained messages. diff --git a/rel/i18n/emqx_authz_api_mnesia.hocon b/rel/i18n/emqx_authz_api_mnesia.hocon index d0021c6a5..0a3b4aba4 100644 --- a/rel/i18n/emqx_authz_api_mnesia.hocon +++ b/rel/i18n/emqx_authz_api_mnesia.hocon @@ -1,10 +1,20 @@ emqx_authz_api_mnesia { action.desc: -"""Authorized action (pub/sub/all)""" +"""Authorized action (publish/subscribe/all)""" action.label: """action""" +qos.desc: +"""QoS of authorized action""" +qos.label: +"""QoS""" + +retain.desc: +"""Retain flag of authorized action""" +retain.label: +"""retain""" + clientid.desc: """ClientID""" clientid.label: From 1ce6a225ae6e57c1b27d9461dcbea6bb0324b334 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Jul 2023 18:39:13 +0300 Subject: [PATCH 66/92] feat(authz): add tests for authz extended actions --- Makefile | 8 ++-- apps/emqx_authz/src/emqx_authz_mongodb.erl | 11 +---- apps/emqx_authz/src/emqx_authz_mysql.erl | 2 - apps/emqx_authz/src/emqx_authz_postgresql.erl | 2 - apps/emqx_authz/src/emqx_authz_redis.erl | 9 +++-- .../emqx_authz/test/emqx_authz_file_SUITE.erl | 14 +++---- .../test/emqx_authz_mysql_SUITE.erl | 34 +++++++++++++--- .../test/emqx_authz_postgresql_SUITE.erl | 13 ++++++ .../test/emqx_authz_rule_raw_SUITE.erl | 14 +++++++ apps/emqx_authz/test/emqx_authz_test_lib.erl | 2 + .../src/bhvrs/emqx_gateway_conn.erl | 4 +- .../test/emqx_coap_SUITE.erl | 40 ++++++++++++++++--- apps/emqx_mysql/src/emqx_mysql.erl | 2 +- 13 files changed, 111 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index 8c5eb3048..7362cdac4 100644 --- a/Makefile +++ b/Makefile @@ -130,14 +130,14 @@ $(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app)))) ct-suite: $(REBAR) merge-config clean-test-cluster-config ifneq ($(TESTCASE),) ifneq ($(GROUP),) - $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) --group $(GROUP) + $(REBAR) ct -c -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) --group $(GROUP) else - $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) + $(REBAR) ct -c -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) endif else ifneq ($(GROUP),) - $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --group $(GROUP) + $(REBAR) ct -c -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --group $(GROUP) else - $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) + $(REBAR) ct -c -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) endif .PHONY: cover diff --git a/apps/emqx_authz/src/emqx_authz_mongodb.erl b/apps/emqx_authz/src/emqx_authz_mongodb.erl index 7adb6d2d9..52a920d3a 100644 --- a/apps/emqx_authz/src/emqx_authz_mongodb.erl +++ b/apps/emqx_authz/src/emqx_authz_mongodb.erl @@ -77,14 +77,7 @@ authorize( } ) -> RenderedFilter = emqx_authz_utils:render_deep(FilterTemplate, Client), - Result = - try - emqx_resource:simple_sync_query(ResourceID, {find, Collection, RenderedFilter, #{}}) - catch - error:Error -> {error, Error} - end, - - case Result of + case emqx_resource:simple_sync_query(ResourceID, {find, Collection, RenderedFilter, #{}}) of {error, Reason} -> ?SLOG(error, #{ msg => "query_mongo_error", @@ -94,8 +87,6 @@ authorize( resource_id => ResourceID }), nomatch; - {ok, []} -> - nomatch; {ok, Rows} -> Rules = lists:flatmap(fun parse_rule/1, Rows), do_authorize(Client, Action, Topic, Rules) diff --git a/apps/emqx_authz/src/emqx_authz_mysql.erl b/apps/emqx_authz/src/emqx_authz_mysql.erl index 01debaea9..a724e451c 100644 --- a/apps/emqx_authz/src/emqx_authz_mysql.erl +++ b/apps/emqx_authz/src/emqx_authz_mysql.erl @@ -86,8 +86,6 @@ authorize( case emqx_resource:simple_sync_query(ResourceID, {prepared_query, ?PREPARE_KEY, RenderParams}) of - {ok, _ColumnNames, []} -> - nomatch; {ok, ColumnNames, Rows} -> do_authorize(Client, Action, Topic, ColumnNames, Rows); {error, Reason} -> diff --git a/apps/emqx_authz/src/emqx_authz_postgresql.erl b/apps/emqx_authz/src/emqx_authz_postgresql.erl index 1b05451cc..f0bdf77be 100644 --- a/apps/emqx_authz/src/emqx_authz_postgresql.erl +++ b/apps/emqx_authz/src/emqx_authz_postgresql.erl @@ -93,8 +93,6 @@ authorize( case emqx_resource:simple_sync_query(ResourceID, {prepared_query, ResourceID, RenderedParams}) of - {ok, _Columns, []} -> - nomatch; {ok, Columns, Rows} -> do_authorize(Client, Action, Topic, column_names(Columns), Rows); {error, Reason} -> diff --git a/apps/emqx_authz/src/emqx_authz_redis.erl b/apps/emqx_authz/src/emqx_authz_redis.erl index 01149b5bb..d163c0d16 100644 --- a/apps/emqx_authz/src/emqx_authz_redis.erl +++ b/apps/emqx_authz/src/emqx_authz_redis.erl @@ -80,8 +80,6 @@ authorize( Vars = emqx_authz_utils:vars_for_rule_query(Client, Action), Cmd = emqx_authz_utils:render_deep(CmdTemplate, Vars), case emqx_resource:simple_sync_query(ResourceID, {cmd, Cmd}) of - {ok, []} -> - nomatch; {ok, Rows} -> do_authorize(Client, Action, Topic, Rows); {error, Reason} -> @@ -108,12 +106,13 @@ do_authorize(Client, Action, Topic, [TopicFilterRaw, RuleEncoded | Tail]) -> {matched, Permission} -> {matched, Permission}; nomatch -> do_authorize(Client, Action, Topic, Tail) catch - error:Reason -> + error:Reason:Stack -> ?SLOG(error, #{ msg => "match_rule_error", reason => Reason, rule_encoded => RuleEncoded, - topic_filter_raw => TopicFilterRaw + topic_filter_raw => TopicFilterRaw, + stacktrace => Stack }), do_authorize(Client, Action, Topic, Tail) end. @@ -148,6 +147,8 @@ parse_rule(Bin) when is_binary(Bin) -> case emqx_utils_json:safe_decode(Bin, [return_maps]) of {ok, Map} when is_map(Map) -> maps:with([<<"qos">>, <<"action">>, <<"retain">>], Map); + {ok, _} -> + error({invalid_topic_rule, Bin, notamap}); {error, Error} -> error({invalid_topic_rule, Bin, Error}) end. diff --git a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl index 0ce788f8d..396679783 100644 --- a/apps/emqx_authz/test/emqx_authz_file_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_file_SUITE.erl @@ -38,21 +38,19 @@ all() -> groups() -> []. -init_per_suite(Config) -> - Config. - -end_per_suite(_Config) -> - ok = emqx_authz_test_lib:restore_authorizers(). - init_per_testcase(TestCase, Config) -> Apps = emqx_cth_suite:start( - [{emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"}, emqx_authz], + [ + {emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"}, + emqx_authz + ], #{work_dir => filename:join(?config(priv_dir, Config), TestCase)} ), [{tc_apps, Apps} | Config]. end_per_testcase(_TestCase, Config) -> - emqx_cth_suite:stop(?config(tc_apps, Config)). + emqx_cth_suite:stop(?config(tc_apps, Config)), + _ = emqx_authz:set_feature_available(rich_actions, true). %%------------------------------------------------------------------------------ %% Testcases diff --git a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl index f31a6ceab..4304dd505 100644 --- a/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl @@ -333,19 +333,41 @@ cases() -> }, #{ name => invalid_query, - setup => [], - query => "SELECT permission, action, topic FROM acl WHER", + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))" + ], + query => "SELECT permission, action, topic FRO", checks => [ {deny, ?AUTHZ_PUBLISH, <<"a">>} ] }, #{ - name => pgsql_error, - setup => [], + name => runtime_error, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))" + ], query => - "SELECT permission, action, topic FROM table_not_exists WHERE username = ${username}", + "SELECT permission, action, topic FROM acl WHERE username = ${username}", checks => [ - {deny, ?AUTHZ_PUBLISH, <<"t">>} + fun() -> + _ = q("DROP TABLE IF EXISTS acl"), + {deny, ?AUTHZ_PUBLISH, <<"t">>} + end + ] + }, + #{ + name => invalid_rule, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + %% 'permit' is invalid value for action + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'permit', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} ] } ]. diff --git a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl index 0c446ee99..a9181879e 100644 --- a/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl @@ -352,6 +352,19 @@ cases() -> checks => [ {deny, ?AUTHZ_PUBLISH, <<"t">>} ] + }, + #{ + name => invalid_rule, + setup => [ + "CREATE TABLE acl(username VARCHAR(255), topic VARCHAR(255), " + "permission VARCHAR(255), action VARCHAR(255))", + %% 'permit' is invalid value for action + "INSERT INTO acl(username, topic, permission, action) VALUES('username', 'a', 'permit', 'publish')" + ], + query => "SELECT permission, action, topic FROM acl WHERE username = ${username}", + checks => [ + {deny, ?AUTHZ_PUBLISH, <<"a">>} + ] } %% TODO: add case for unknown variables after fixing EMQX-10400 ]. diff --git a/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl b/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl index 8b097c3fd..798661b53 100644 --- a/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl +++ b/apps/emqx_authz/test/emqx_authz_rule_raw_SUITE.erl @@ -181,6 +181,14 @@ ok_cases() -> expected_rule_with_qos_retain([0, 1, 2], all), rule_with_raw_qos_retain(#{}) }, + { + expected_rule_with_qos_retain([0, 1, 2], true), + rule_with_raw_qos_retain(#{<<"retain">> => <<"1">>}) + }, + { + expected_rule_with_qos_retain([0, 1, 2], false), + rule_with_raw_qos_retain(#{<<"retain">> => <<"0">>}) + }, %% Qos { expected_rule_with_qos_retain([2], all), @@ -254,6 +262,12 @@ error_rich_action_cases() -> <<"topics">> => [], <<"action">> => <<"publish">>, <<"retain">> => 3 + }, + #{ + <<"permission">> => <<"allow">>, + <<"topics">> => [], + <<"action">> => <<"publish">>, + <<"qos">> => [<<"3">>] } ]. diff --git a/apps/emqx_authz/test/emqx_authz_test_lib.erl b/apps/emqx_authz/test/emqx_authz_test_lib.erl index ea62c5b71..33035c766 100644 --- a/apps/emqx_authz/test/emqx_authz_test_lib.erl +++ b/apps/emqx_authz/test/emqx_authz_test_lib.erl @@ -107,6 +107,8 @@ run_checks(#{checks := Checks} = Case) -> Checks ). +run_check(ClientInfo, Fun) when is_function(Fun, 0) -> + run_check(ClientInfo, Fun()); run_check(ClientInfo, {ExpectedPermission, Action, Topic}) -> ?assertEqual( ExpectedPermission, diff --git a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl index 52f96bcd2..9f3344e95 100644 --- a/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl +++ b/apps/emqx_gateway/src/bhvrs/emqx_gateway_conn.erl @@ -715,13 +715,13 @@ parse_incoming( NState = State#state{parse_state = NParseState}, parse_incoming(Rest, [Packet | Packets], NState) catch - error:Reason:Stk -> + error:Reason:Stack -> ?SLOG(error, #{ msg => "parse_frame_failed", at_state => ParseState, input_bytes => Data, reason => Reason, - stacktrace => Stk + stacktrace => Stack }), {[{frame_error, Reason} | Packets], State} end. diff --git a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl index ce809184e..091978172 100644 --- a/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl +++ b/apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl @@ -59,15 +59,14 @@ init_per_suite(Config) -> application:load(emqx_gateway_coap), ok = emqx_common_test_helpers:load_config(emqx_gateway_schema, ?CONF_DEFAULT), emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]), - ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), Config. end_per_suite(_) -> - meck:unload(emqx_access_control), {ok, _} = emqx:remove_config([<<"gateway">>, <<"coap">>]), emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]). init_per_testcase(t_connection_with_authn_failed, Config) -> + ok = meck:new(emqx_access_control, [passthrough]), ok = meck:expect( emqx_access_control, authenticate, @@ -75,12 +74,11 @@ init_per_testcase(t_connection_with_authn_failed, Config) -> ), Config; init_per_testcase(_, Config) -> + ok = meck:new(emqx_access_control, [passthrough]), Config. -end_per_testcase(t_connection_with_authn_failed, Config) -> - ok = meck:delete(emqx_access_control, authenticate, 1), - Config; end_per_testcase(_, Config) -> + ok = meck:unload(emqx_access_control), Config. default_config() -> @@ -213,6 +211,38 @@ t_publish(_) -> end, with_connection(Topics, Action). +t_publish_with_retain_qos_expiry(_) -> + _ = meck:expect( + emqx_access_control, + authorize, + fun(_, #{action_type := publish, qos := 1, retain := true}, _) -> + allow + end + ), + + Topics = [<<"abc">>], + Action = fun(Topic, Channel, Token) -> + Payload = <<"123">>, + URI = pubsub_uri(binary_to_list(Topic), Token) ++ "&retain=true&qos=1&expiry=60", + + %% Sub topic first + emqx:subscribe(Topic), + + Req = make_req(post, Payload), + {ok, changed, _} = do_request(Channel, URI, Req), + + receive + {deliver, Topic, Msg} -> + ?assertEqual(Topic, Msg#message.topic), + ?assertEqual(Payload, Msg#message.payload) + after 500 -> + ?assert(false) + end + end, + with_connection(Topics, Action), + + _ = meck:validate(emqx_access_control). + t_subscribe(_) -> %% can subscribe to a normal topic Topics = [ diff --git a/apps/emqx_mysql/src/emqx_mysql.erl b/apps/emqx_mysql/src/emqx_mysql.erl index 2a4db3147..c9273f3f1 100644 --- a/apps/emqx_mysql/src/emqx_mysql.erl +++ b/apps/emqx_mysql/src/emqx_mysql.erl @@ -361,7 +361,7 @@ prepare_sql_to_conn(Conn, [{Key, SQL} | PrepareList]) when is_pid(Conn) -> ?SLOG(error, LogMeta#{result => failed, reason => Reason}), {error, undefined_table}; {error, Reason} -> - % FIXME: we should try to differ on transient failers and + % FIXME: we should try to differ on transient failures and % syntax failures. Retrying syntax failures is not very productive. ?SLOG(error, LogMeta#{result => failed, reason => Reason}), {error, Reason} From 6db02d6b49b7ca69cef4c3c91f7f4228ad3d3af9 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Mon, 3 Jul 2023 18:40:17 +0300 Subject: [PATCH 67/92] feat(authz): enable feature for both CE and EE --- apps/emqx_authz/src/emqx_authz.erl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index b5f1a1298..830db21b7 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -528,16 +528,8 @@ read_acl_file(#{<<"path">> := Path} = Source) -> %% Extednded Features %%------------------------------------------------------------------------------ --if(?EMQX_RELEASE_EDITION == ee). - -define(DEFAULT_RICH_ACTIONS, true). --else. - --define(DEFAULT_RICH_ACTIONS, false). - --endif. - -define(FEATURE_KEY(_NAME_), {?MODULE, _NAME_}). feature_available(rich_actions) -> From 77895f2555e944712d5409ddf81d0b21ac8faded Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 7 Jul 2023 16:20:55 +0300 Subject: [PATCH 68/92] feat(authz): fix typos and style for Retain & QoS authz feature Co-authored-by: Thales Macedo Garitezi --- Makefile | 8 ++++---- apps/emqx/src/emqx_channel.erl | 2 +- apps/emqx_authz/include/emqx_authz.hrl | 24 ++++++++++++------------ apps/emqx_authz/src/emqx_authz.erl | 2 +- apps/emqx_authz/src/emqx_authz_rule.erl | 2 +- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 7362cdac4..8c5eb3048 100644 --- a/Makefile +++ b/Makefile @@ -130,14 +130,14 @@ $(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app)))) ct-suite: $(REBAR) merge-config clean-test-cluster-config ifneq ($(TESTCASE),) ifneq ($(GROUP),) - $(REBAR) ct -c -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) --group $(GROUP) + $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) --group $(GROUP) else - $(REBAR) ct -c -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) + $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --case $(TESTCASE) endif else ifneq ($(GROUP),) - $(REBAR) ct -c -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --group $(GROUP) + $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) --group $(GROUP) else - $(REBAR) ct -c -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) + $(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE) endif .PHONY: cover diff --git a/apps/emqx/src/emqx_channel.erl b/apps/emqx/src/emqx_channel.erl index 01af1a7b5..5e594d35f 100644 --- a/apps/emqx/src/emqx_channel.erl +++ b/apps/emqx/src/emqx_channel.erl @@ -1840,7 +1840,7 @@ check_pub_alias(_Packet, _Channel) -> ok. %%-------------------------------------------------------------------- -%% Athorization action +%% Authorization action authz_action(#mqtt_packet{ header = #mqtt_packet_header{qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{} diff --git a/apps/emqx_authz/include/emqx_authz.hrl b/apps/emqx_authz/include/emqx_authz.hrl index 8676da134..5cab24fab 100644 --- a/apps/emqx_authz/include/emqx_authz.hrl +++ b/apps/emqx_authz/include/emqx_authz.hrl @@ -49,12 +49,12 @@ username => user1, rules => [ #{ - topic => <<"test/toopic/1">>, + topic => <<"test/topic/1">>, permission => <<"allow">>, action => <<"publish">> }, #{ - topic => <<"test/toopic/2">>, + topic => <<"test/topic/2">>, permission => <<"allow">>, action => <<"subscribe">> }, @@ -64,14 +64,14 @@ action => <<"all">> }, #{ - topic => <<"test/toopic/3">>, + topic => <<"test/topic/3">>, permission => <<"allow">>, action => <<"publish">>, qos => [<<"1">>], retain => <<"true">> }, #{ - topic => <<"test/toopic/4">>, + topic => <<"test/topic/4">>, permission => <<"allow">>, action => <<"publish">>, qos => [<<"0">>, <<"1">>, <<"2">>], @@ -83,12 +83,12 @@ clientid => client1, rules => [ #{ - topic => <<"test/toopic/1">>, + topic => <<"test/topic/1">>, permission => <<"allow">>, action => <<"publish">> }, #{ - topic => <<"test/toopic/2">>, + topic => <<"test/topic/2">>, permission => <<"allow">>, action => <<"subscribe">> }, @@ -98,14 +98,14 @@ action => <<"all">> }, #{ - topic => <<"test/toopic/3">>, + topic => <<"test/topic/3">>, permission => <<"allow">>, action => <<"publish">>, qos => [<<"1">>], retain => <<"true">> }, #{ - topic => <<"test/toopic/4">>, + topic => <<"test/topic/4">>, permission => <<"allow">>, action => <<"publish">>, qos => [<<"0">>, <<"1">>, <<"2">>], @@ -116,12 +116,12 @@ -define(ALL_RULES_EXAMPLE, #{ rules => [ #{ - topic => <<"test/toopic/1">>, + topic => <<"test/topic/1">>, permission => <<"allow">>, action => <<"publish">> }, #{ - topic => <<"test/toopic/2">>, + topic => <<"test/topic/2">>, permission => <<"allow">>, action => <<"subscribe">> }, @@ -131,14 +131,14 @@ action => <<"all">> }, #{ - topic => <<"test/toopic/3">>, + topic => <<"test/topic/3">>, permission => <<"allow">>, action => <<"publish">>, qos => [<<"1">>], retain => <<"true">> }, #{ - topic => <<"test/toopic/4">>, + topic => <<"test/topic/4">>, permission => <<"allow">>, action => <<"publish">>, qos => [<<"0">>, <<"1">>, <<"2">>], diff --git a/apps/emqx_authz/src/emqx_authz.erl b/apps/emqx_authz/src/emqx_authz.erl index 830db21b7..0419bcf72 100644 --- a/apps/emqx_authz/src/emqx_authz.erl +++ b/apps/emqx_authz/src/emqx_authz.erl @@ -525,7 +525,7 @@ read_acl_file(#{<<"path">> := Path} = Source) -> maps:remove(<<"path">>, Source#{<<"rules">> => Rules}). %%------------------------------------------------------------------------------ -%% Extednded Features +%% Extended Features %%------------------------------------------------------------------------------ -define(DEFAULT_RICH_ACTIONS, true). diff --git a/apps/emqx_authz/src/emqx_authz_rule.erl b/apps/emqx_authz/src/emqx_authz_rule.erl index a59679a8d..6e13cac91 100644 --- a/apps/emqx_authz/src/emqx_authz_rule.erl +++ b/apps/emqx_authz/src/emqx_authz_rule.erl @@ -60,7 +60,7 @@ -type rule() :: {permission(), who_condition(), action_condition(), topic_condition()}. --type qos() :: 0..2. +-type qos() :: emqx_types:qos(). -type retain() :: boolean(). -type action() :: #{action_type := subscribe, qos := qos()} From 19f9fc508942ff89e08177951b5ba841c9e921c6 Mon Sep 17 00:00:00 2001 From: Ilya Averyanov Date: Fri, 7 Jul 2023 19:41:34 +0300 Subject: [PATCH 69/92] feat(authz): bump app versions --- apps/emqx_authz/src/emqx_authz.app.src | 2 +- apps/emqx_mysql/src/emqx_mysql.app.src | 2 +- apps/emqx_rule_engine/src/emqx_rule_engine.app.src | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_authz/src/emqx_authz.app.src b/apps/emqx_authz/src/emqx_authz.app.src index e9654557e..29a035ab3 100644 --- a/apps/emqx_authz/src/emqx_authz.app.src +++ b/apps/emqx_authz/src/emqx_authz.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_authz, [ {description, "An OTP application"}, - {vsn, "0.1.23"}, + {vsn, "0.1.24"}, {registered, []}, {mod, {emqx_authz_app, []}}, {applications, [ diff --git a/apps/emqx_mysql/src/emqx_mysql.app.src b/apps/emqx_mysql/src/emqx_mysql.app.src index c0f8ec7e7..df4846356 100644 --- a/apps/emqx_mysql/src/emqx_mysql.app.src +++ b/apps/emqx_mysql/src/emqx_mysql.app.src @@ -1,6 +1,6 @@ {application, emqx_mysql, [ {description, "EMQX MySQL Database Connector"}, - {vsn, "0.1.0"}, + {vsn, "0.1.1"}, {registered, []}, {applications, [ kernel, diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src index ccd436d86..f0388631f 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.app.src +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.app.src @@ -2,7 +2,7 @@ {application, emqx_rule_engine, [ {description, "EMQX Rule Engine"}, % strict semver, bump manually! - {vsn, "5.0.20"}, + {vsn, "5.0.21"}, {modules, []}, {registered, [emqx_rule_engine_sup, emqx_rule_engine]}, {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]}, From 3bc419ee64a33b9ee01ddff47075dc41ed64909a Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 6 Jul 2023 15:25:50 +0200 Subject: [PATCH 70/92] fix(emqx_gateway): return 404 for unknown listener id --- apps/emqx_gateway/src/emqx_gateway_api_listeners.erl | 9 +++++++-- apps/emqx_gateway/src/emqx_gateway_http.erl | 2 +- apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl | 11 +++++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl index e62923bc2..046e23300 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_listeners.erl @@ -112,8 +112,13 @@ listeners(post, #{bindings := #{name := Name0}, body := LConf}) -> listeners_insta(delete, #{bindings := #{name := Name0, id := ListenerId}}) -> with_gateway(Name0, fun(_GwName, _) -> - ok = emqx_gateway_http:remove_listener(ListenerId), - {204} + case emqx_gateway_conf:listener(ListenerId) of + {ok, _Listener} -> + ok = emqx_gateway_http:remove_listener(ListenerId), + {204}; + {error, not_found} -> + return_http_error(404, "Listener not found") + end end); listeners_insta(get, #{bindings := #{name := Name0, id := ListenerId}}) -> with_gateway(Name0, fun(_GwName, _) -> diff --git a/apps/emqx_gateway/src/emqx_gateway_http.erl b/apps/emqx_gateway/src/emqx_gateway_http.erl index 58c201c75..2186ac3d7 100644 --- a/apps/emqx_gateway/src/emqx_gateway_http.erl +++ b/apps/emqx_gateway/src/emqx_gateway_http.erl @@ -550,7 +550,7 @@ with_gateway(GwName0, Fun) -> return_http_error(400, [K, " is required"]); %% Exceptions from emqx_gateway_utils:parse_listener_id/1 error:{invalid_listener_id, Id} -> - return_http_error(400, ["Invalid listener id: ", Id]); + return_http_error(404, ["Listener not found: ", Id]); %% Exceptions from emqx:get_config/1 error:{config_not_found, Path0} -> Path = lists:concat( diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index f1cfd26d0..e486c8c16 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -409,6 +409,7 @@ t_listeners_tcp(_) -> {204, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"), {404, _} = request(get, "/gateways/stomp/listeners/stomp:tcp:def"), + {404, _} = request(delete, "/gateways/stomp/listeners/stomp:tcp:def"), ok. t_listeners_max_conns(_) -> @@ -480,9 +481,19 @@ t_listeners_authn(_) -> {200, ConfResp3} = request(get, Path), assert_confs(AuthConf2, ConfResp3), + {404, _} = request(get, Path ++ "/users/not_exists"), + {404, _} = request(delete, Path ++ "/users/not_exists"), + {204, _} = request(delete, Path), %% FIXME: 204? {204, _} = request(get, Path), + + BadPath = "/gateways/stomp/listeners/stomp:tcp:not_exists/authentication/users/foo", + {404, _} = request(get, BadPath), + {404, _} = request(delete, BadPath), + + {404, _} = request(get, "/gateways/stomp/listeners/not_exists"), + {404, _} = request(delete, "/gateways/stomp/listeners/not_exists"), ok. t_listeners_authn_data_mgmt(_) -> From d65d690c175fe3e8cdc11a2d6eaa53c392654bb4 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Thu, 6 Jul 2023 15:26:15 +0200 Subject: [PATCH 71/92] fix(emqx_gateway): return 404 for unknown client id --- .../src/emqx_gateway_api_clients.erl | 48 +++++++++++++++++-- .../test/emqx_gateway_api_SUITE.erl | 41 ++++++++++++++++ .../test/emqx_gateway_cli_SUITE.erl | 16 +------ .../test/emqx_gateway_test_utils.erl | 16 +++++++ 4 files changed, 102 insertions(+), 19 deletions(-) diff --git a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl index 8037f4197..8cfcb70e6 100644 --- a/apps/emqx_gateway/src/emqx_gateway_api_clients.erl +++ b/apps/emqx_gateway/src/emqx_gateway_api_clients.erl @@ -57,7 +57,8 @@ qs2ms/2, run_fuzzy_filter/2, format_channel_info/1, - format_channel_info/2 + format_channel_info/2, + client_info_mountpoint/1 ]). -define(TAGS, [<<"Gateway Clients">>]). @@ -177,8 +178,12 @@ clients_insta(delete, #{ } }) -> with_gateway(Name0, fun(GwName, _) -> - _ = emqx_gateway_http:kickout_client(GwName, ClientId), - {204} + case emqx_gateway_http:kickout_client(GwName, ClientId) of + {error, not_found} -> + return_http_error(404, "Client not found"); + _ -> + {204} + end end). %% List the established subscriptions with mountpoint @@ -234,8 +239,13 @@ subscriptions(delete, #{ } }) -> with_gateway(Name0, fun(GwName, _) -> - _ = emqx_gateway_http:client_unsubscribe(GwName, ClientId, Topic), - {204} + case lookup_topic(GwName, ClientId, Topic) of + {ok, _} -> + _ = emqx_gateway_http:client_unsubscribe(GwName, ClientId, Topic), + {204}; + {error, not_found} -> + return_http_error(404, "Resource not found") + end end). %%-------------------------------------------------------------------- @@ -260,6 +270,34 @@ extra_sub_props(Props) -> #{subid => maps:get(<<"subid">>, Props, undefined)} ). +lookup_topic(GwName, ClientId, Topic) -> + Mountpoints = emqx_gateway_http:lookup_client( + GwName, + ClientId, + {?MODULE, client_info_mountpoint} + ), + case emqx_gateway_http:list_client_subscriptions(GwName, ClientId) of + {ok, Subscriptions} -> + case + [ + S + || S = #{topic := Topic0} <- Subscriptions, + Mountpoint <- Mountpoints, + Topic0 == emqx_mountpoint:mount(Mountpoint, Topic) + ] + of + [] -> + {error, not_found}; + Filtered -> + {ok, Filtered} + end; + Error -> + Error + end. + +client_info_mountpoint({_, #{clientinfo := #{mountpoint := Mountpoint}}, _}) -> + Mountpoint. + %%-------------------------------------------------------------------- %% QueryString to MatchSpec diff --git a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl index e486c8c16..a3fe39852 100644 --- a/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl @@ -586,6 +586,47 @@ t_listeners_authn_data_mgmt(_) -> ok. +t_clients(_) -> + GwConf = #{ + name => <<"mqttsn">>, + gateway_id => 1, + broadcast => true, + predefined => [#{id => 1, topic => <<"t/a">>}], + enable_qos3 => true, + listeners => [ + #{name => <<"def">>, type => <<"udp">>, bind => <<"1884">>} + ] + }, + init_gw("mqttsn", GwConf), + Path = "/gateways/mqttsn/clients", + MyClient = Path ++ "/my_client", + MyClientSubscriptions = MyClient ++ "/subscriptions", + {200, NoClients} = request(get, Path), + ?assertMatch(#{data := []}, NoClients), + + ClientSocket = emqx_gateway_test_utils:sn_client_connect(<<"my_client">>), + {200, _} = request(get, MyClient), + {200, Clients} = request(get, Path), + ?assertMatch(#{data := [#{clientid := <<"my_client">>}]}, Clients), + + {201, _} = request(post, MyClientSubscriptions, #{topic => <<"test/topic">>}), + {200, Subscriptions} = request(get, MyClientSubscriptions), + ?assertMatch([#{topic := <<"test/topic">>}], Subscriptions), + {204, _} = request(delete, MyClientSubscriptions ++ "/test%2Ftopic"), + {200, []} = request(get, MyClientSubscriptions), + {404, _} = request(delete, MyClientSubscriptions ++ "/test%2Ftopic"), + + {204, _} = request(delete, MyClient), + {404, _} = request(delete, MyClient), + {404, _} = request(get, MyClient), + {404, _} = request(get, MyClientSubscriptions), + {404, _} = request(post, MyClientSubscriptions, #{topic => <<"foo">>}), + {404, _} = request(delete, MyClientSubscriptions ++ "/topic"), + {200, NoClients2} = request(get, Path), + ?assertMatch(#{data := []}, NoClients2), + emqx_gateway_test_utils:sn_client_disconnect(ClientSocket), + ok. + t_authn_fuzzy_search(_) -> init_gw("stomp"), AuthConf = #{ diff --git a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl index 641528eda..fdaa55ab8 100644 --- a/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl +++ b/apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl @@ -54,6 +54,8 @@ end). "}\n" ). +-import(emqx_gateway_test_utils, [sn_client_connect/1, sn_client_disconnect/1]). + %%-------------------------------------------------------------------- %% Setup %%-------------------------------------------------------------------- @@ -303,17 +305,3 @@ acc_print(Acc) -> after 200 -> Acc end. - -sn_client_connect(ClientId) -> - {ok, Socket} = gen_udp:open(0, [binary]), - _ = emqx_sn_protocol_SUITE:send_connect_msg(Socket, ClientId), - ?assertEqual( - <<3, 16#05, 0>>, - emqx_sn_protocol_SUITE:receive_response(Socket) - ), - Socket. - -sn_client_disconnect(Socket) -> - _ = emqx_sn_protocol_SUITE:send_disconnect_msg(Socket, undefined), - gen_udp:close(Socket), - ok. diff --git a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl index 2e8a3a583..950ae1bcf 100644 --- a/apps/emqx_gateway/test/emqx_gateway_test_utils.erl +++ b/apps/emqx_gateway/test/emqx_gateway_test_utils.erl @@ -19,6 +19,8 @@ -compile(export_all). -compile(nowarn_export_all). +-include_lib("eunit/include/eunit.hrl"). + assert_confs(Expected0, Effected) -> Expected = maybe_unconvert_listeners(Expected0), case do_assert_confs(root, Expected, Effected) of @@ -181,3 +183,17 @@ url(Path, Qs) -> auth(Headers) -> [emqx_mgmt_api_test_util:auth_header_() | Headers]. + +sn_client_connect(ClientId) -> + {ok, Socket} = gen_udp:open(0, [binary]), + _ = emqx_sn_protocol_SUITE:send_connect_msg(Socket, ClientId), + ?assertEqual( + <<3, 16#05, 0>>, + emqx_sn_protocol_SUITE:receive_response(Socket) + ), + Socket. + +sn_client_disconnect(Socket) -> + _ = emqx_sn_protocol_SUITE:send_disconnect_msg(Socket, undefined), + gen_udp:close(Socket), + ok. From 3fd28f9e182ff34d96d85e5f01c6885ab06e562e Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 7 Jul 2023 11:57:33 +0200 Subject: [PATCH 72/92] fix(emqx_management): return 404 for unknown listener id --- .../src/emqx_mgmt_api_listeners.erl | 44 +++++++++---------- .../test/emqx_mgmt_api_listeners_SUITE.erl | 7 ++- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index dd9013b16..8c20cbab8 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -45,14 +45,10 @@ update/3 ]). --include_lib("emqx/include/emqx.hrl"). -include_lib("hocon/include/hoconsc.hrl"). --define(NODE_LISTENER_NOT_FOUND, <<"Node name or listener id not found">>). --define(NODE_NOT_FOUND_OR_DOWN, <<"Node not found or Down">>). -define(LISTENER_NOT_FOUND, <<"Listener id not found">>). -define(LISTENER_ID_INCONSISTENT, <<"Path and body's listener id not match">>). --define(ADDR_PORT_INUSE, <<"Addr port in use">>). namespace() -> "listeners". @@ -156,7 +152,7 @@ schema("/listeners/:id") -> parameters => [?R_REF(listener_id)], responses => #{ 204 => <<"Listener deleted">>, - 400 => error_codes(['BAD_REQUEST']) + 404 => error_codes(['BAD_LISTENER_ID']) } } }; @@ -405,20 +401,8 @@ list_listeners(get, #{query_string := Query}) -> list_listeners(post, #{body := Body}) -> create_listener(Body). -crud_listeners_by_id(get, #{bindings := #{id := Id0}}) -> - Listeners = - [ - Conf#{ - <<"id">> => Id, - <<"type">> => Type, - <<"bind">> := iolist_to_binary( - emqx_listeners:format_bind(maps:get(<<"bind">>, Conf)) - ) - } - || {Id, Type, Conf} <- emqx_listeners:list_raw(), - Id =:= Id0 - ], - case Listeners of +crud_listeners_by_id(get, #{bindings := #{id := Id}}) -> + case find_listeners_by_id(Id) of [] -> {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}}; [L] -> {200, L} end; @@ -449,9 +433,12 @@ crud_listeners_by_id(post, #{body := Body}) -> create_listener(Body); crud_listeners_by_id(delete, #{bindings := #{id := Id}}) -> {ok, #{type := Type, name := Name}} = emqx_listeners:parse_listener_id(Id), - case ensure_remove(Type, Name) of - {ok, _} -> {204}; - {error, Reason} -> {400, #{code => 'BAD_REQUEST', message => err_msg(Reason)}} + case find_listeners_by_id(Id) of + [_L] -> + {ok, _Res} = ensure_remove(Type, Name), + {204}; + [] -> + {404, #{code => 'BAD_LISTENER_ID', message => ?LISTENER_NOT_FOUND}} end. parse_listener_conf(Conf0) -> @@ -585,6 +572,19 @@ do_list_listeners() -> <<"listeners">> => Listeners }. +find_listeners_by_id(Id) -> + [ + Conf#{ + <<"id">> => Id0, + <<"type">> => Type, + <<"bind">> := iolist_to_binary( + emqx_listeners:format_bind(maps:get(<<"bind">>, Conf)) + ) + } + || {Id0, Type, Conf} <- emqx_listeners:list_raw(), + Id0 =:= Id + ]. + wrap_rpc({badrpc, Reason}) -> {error, Reason}; wrap_rpc(Res) -> diff --git a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl index 862d81ab8..96ec8f2de 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl @@ -399,12 +399,15 @@ crud_listeners_by_id(ListenerId, NewListenerId, MinListenerId, BadId, Type, Port ?assertEqual([], delete(MinPath)), ?assertEqual({error, not_found}, is_running(NewListenerId)), ?assertMatch({error, {"HTTP/1.1", 404, _}}, request(get, NewPath, [], [])), - ?assertEqual([], delete(NewPath)), + ?assertMatch({error, {"HTTP/1.1", 404, _}}, request(delete, NewPath, [], [])), ok. t_delete_nonexistent_listener(Config) when is_list(Config) -> NonExist = emqx_mgmt_api_test_util:api_path(["listeners", "tcp:nonexistent"]), - ?assertEqual([], delete(NonExist)), + ?assertMatch( + {error, {_, 404, _}}, + request(delete, NonExist, [], []) + ), ok. t_action_listeners(Config) when is_list(Config) -> From 80e4ffff752918fa4db08091a7eb8b2bdf27b576 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 7 Jul 2023 13:17:52 +0200 Subject: [PATCH 73/92] fix(emqx_management): return 404 if plugin does not exist --- apps/emqx_management/src/emqx_mgmt_api_plugins.erl | 3 +++ apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl index f50e44771..3db0c42fb 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_plugins.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_plugins.erl @@ -132,6 +132,7 @@ schema("/plugins/:name") -> parameters => [hoconsc:ref(name)], responses => #{ 204 => <<"Uninstall successfully">>, + 400 => emqx_dashboard_swagger:error_codes(['PARAM_ERROR'], <<"Bad parameter">>), 404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Plugin Not Found">>) } } @@ -484,6 +485,8 @@ ensure_action(Name, restart) -> return(Code, ok) -> {Code}; +return(_, {error, #{error := "bad_info_file", return := {enoent, _} = Reason}}) -> + {404, #{code => 'NOT_FOUND', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}; return(_, {error, Reason}) -> {400, #{code => 'PARAM_ERROR', message => iolist_to_binary(io_lib:format("~p", [Reason]))}}. diff --git a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl index 62fed8211..ba613abc4 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl @@ -133,6 +133,14 @@ t_bad_plugin(Config) -> ) ). +t_delete_non_existing(_Config) -> + Path = emqx_mgmt_api_test_util:api_path(["plugins", "non_exists"]), + ?assertMatch( + {error, {_, 404, _}}, + emqx_mgmt_api_test_util:request_api(delete, Path) + ), + ok. + list_plugins() -> Path = emqx_mgmt_api_test_util:api_path(["plugins"]), case emqx_mgmt_api_test_util:request_api(get, Path) of From 1110b5d8f5768e314ffcb1570b7d8beb48028f60 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 7 Jul 2023 13:31:57 +0200 Subject: [PATCH 74/92] fix(emqx_retainer): return 404 in delete if topic not found --- apps/emqx_retainer/src/emqx_retainer.app.src | 2 +- apps/emqx_retainer/src/emqx_retainer_api.erl | 13 +++++++++++-- apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl | 1 + 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/emqx_retainer/src/emqx_retainer.app.src b/apps/emqx_retainer/src/emqx_retainer.app.src index d13359509..f117fda05 100644 --- a/apps/emqx_retainer/src/emqx_retainer.app.src +++ b/apps/emqx_retainer/src/emqx_retainer.app.src @@ -2,7 +2,7 @@ {application, emqx_retainer, [ {description, "EMQX Retainer"}, % strict semver, bump manually! - {vsn, "5.0.14"}, + {vsn, "5.0.15"}, {modules, []}, {registered, [emqx_retainer_sup]}, {applications, [kernel, stdlib, emqx, emqx_ctl]}, diff --git a/apps/emqx_retainer/src/emqx_retainer_api.erl b/apps/emqx_retainer/src/emqx_retainer_api.erl index 7b1337140..3274f0e4c 100644 --- a/apps/emqx_retainer/src/emqx_retainer_api.erl +++ b/apps/emqx_retainer/src/emqx_retainer_api.erl @@ -102,6 +102,7 @@ schema(?PREFIX ++ "/message/:topic") -> parameters => parameters(), responses => #{ 204 => <<>>, + 404 => error_codes(['NOT_FOUND'], ?DESC(message_not_exist)), 400 => error_codes( ['BAD_REQUEST'], ?DESC(unsupported_backend) @@ -187,8 +188,16 @@ with_topic(get, #{bindings := Bindings}) -> end; with_topic(delete, #{bindings := Bindings}) -> Topic = maps:get(topic, Bindings), - emqx_retainer_mnesia:delete_message(undefined, Topic), - {204}. + case emqx_retainer_mnesia:page_read(undefined, Topic, 1, 1) of + {ok, []} -> + {404, #{ + code => <<"NOT_FOUND">>, + message => <<"Viewed message doesn't exist">> + }}; + {ok, _} -> + emqx_retainer_mnesia:delete_message(undefined, Topic), + {204} + end. format_message(#message{ id = ID, diff --git a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl index 61eee0510..d00ade556 100644 --- a/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl +++ b/apps/emqx_retainer/test/emqx_retainer_api_SUITE.erl @@ -218,6 +218,7 @@ t_lookup_and_delete(_) -> {ok, []} = request_api(delete, API), {error, {"HTTP/1.1", 404, "Not Found"}} = request_api(get, API), + {error, {"HTTP/1.1", 404, "Not Found"}} = request_api(delete, API), ok = emqtt:disconnect(C1). From da052e0a5e1720c63930232701f47477ac5f1e09 Mon Sep 17 00:00:00 2001 From: Stefan Strigler Date: Fri, 7 Jul 2023 16:38:27 +0200 Subject: [PATCH 75/92] chore: add changelog --- changes/ce/fix-11211.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-11211.en.md diff --git a/changes/ce/fix-11211.en.md b/changes/ce/fix-11211.en.md new file mode 100644 index 000000000..0b69fefca --- /dev/null +++ b/changes/ce/fix-11211.en.md @@ -0,0 +1 @@ +Consistently return `404` for `DELETE` operations on non-existent resources. From a3de04ebd2fc5568db69cb6a76bfbc97291c47ad Mon Sep 17 00:00:00 2001 From: firest Date: Fri, 7 Jul 2023 16:08:13 +0800 Subject: [PATCH 76/92] chore: add example for DynamoDB template --- rel/i18n/emqx_bridge_dynamo.hocon | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rel/i18n/emqx_bridge_dynamo.hocon b/rel/i18n/emqx_bridge_dynamo.hocon index 46ae9d1bb..899a47c75 100644 --- a/rel/i18n/emqx_bridge_dynamo.hocon +++ b/rel/i18n/emqx_bridge_dynamo.hocon @@ -35,7 +35,9 @@ local_topic.label: """Local Topic""" template.desc: -"""Template, the default value is empty. When this value is empty the whole message will be stored in the database""" +"""Template, the default value is empty. When this value is empty the whole message will be stored in the database.
+The template can be any valid json with placeholders and make sure all keys for table are here, example:
+ {"id" : ${id}, "clientid" : ${clientid}, "data" : ${payload}}""" template.label: """Template""" From a31b5f1ac11e025d4da303cf05b27f612f68c9dc Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Sun, 9 Jul 2023 19:18:04 +0200 Subject: [PATCH 77/92] chore: add scripts/dev-cluster-host.sh --- scripts/dev-cluster-host.sh | 125 ++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100755 scripts/dev-cluster-host.sh diff --git a/scripts/dev-cluster-host.sh b/scripts/dev-cluster-host.sh new file mode 100755 index 000000000..9ce083707 --- /dev/null +++ b/scripts/dev-cluster-host.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +set -euo pipefail + +## This starts configurable number of core and replicant nodes on the same host (not in docker). +## The nodes are named as core1, core2, replicant3, replicant4, ... where the number monotically increases. +## The number in node name is used as an offset for ekka to avoid clashing (see ekka_dist:offset/1). +## Nodes are started on loopback addresses starting from 127.0.0.1. +## The script uses sudo to add loopback aliases. +## The boot script is ./_build/emqx/rel/emqx/bin/emqx. +## The data and log directories are configured to use ./tmp/ + +# ensure dir +cd -P -- "$(dirname -- "$0")/../" + +help() { + echo + echo "start | stop" + echo "-h|--help: To display this usage info" + echo "-n|--nodes: total number of nodes to start (default: 2)" + echo "-c|--core_nodes: number of core nodes to start (default: 1)" + echo "-b|--boot: boot script (default: ./_build/emqx/rel/emqx/bin/emqx)" +} + +CMD="$1" +shift || true + +export EMQX_NODE__COOKIE=test +BOOT_SCRIPT='./_build/emqx/rel/emqx/bin/emqx' +NODES=2 +CORE_NODES=1 + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + -n|--nodes) + NODES="$2" + shift 2 + ;; + -c|--core-nodes) + CORE_NODES="$2" + shift 2 + ;; + -b|--boot) + BOOT_SCRIPT="$2" + shift 2 + ;; + *) + echo "unknown option $1" + exit 1 + ;; + esac +done + +REPLICANT_NODES=$((NODES - CORE_NODES)) + +# cannot use the same node name even IPs are different because Erlang distribution listens on 0.0.0.0 +CORE_IDS=() +REPLICANT_IDS=() +SEEDS_ARRAY=() +for i in $(seq 1 "$CORE_NODES"); do + SEEDS_ARRAY+=("core${i}@127.0.0.$i") + CORE_IDS+=("$i") +done +for i in $(seq "$((CORE_NODES+1))" "$((CORE_NODES+REPLICANT_NODES))"); do + REPLICANT_IDS+=("$i") +done + +SEEDS="$(IFS=,; echo "${SEEDS_ARRAY[*]}")" + +if [ "$CMD" = "stop" ]; then + for id in "${REPLICANT_IDS[@]}"; do + env EMQX_NODE_NAME="replicant${id}@127.0.0.$id" "$BOOT_SCRIPT" stop || true + done + for id in "${CORE_IDS[@]}"; do + env EMQX_NODE_NAME="core${id}@127.0.0.$id" "$BOOT_SCRIPT" stop || true + done + exit 0 +fi + +start_cmd() { + local role="$1" + local id="$2" + local ip="127.0.0.$id" + local nodename="$role$id" + local nodehome="$(pwd)/tmp/$nodename" + mkdir -p "${nodehome}/data" "${nodehome}/log" + cat <<-EOF +env DEBUG="${DEBUG:-0}" \ +EMQX_NODE_NAME="$nodename@$ip" \ +EMQX_CLUSTER__STATIC__SEEDS="$SEEDS" \ +EMQX_CLUSTER__DISCOVERY_STRATEGY=static \ +EMQX_NODE__DB_ROLE="$role" \ +EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \ +EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="${nodehome}/log/emqx.log" \ +EMQX_LOG_DIR="${nodehome}/log" \ +EMQX_NODE__DATA_DIR="${nodehome}/data" \ +EMQX_LISTENERS__TCP__DEFAULT__BIND="$ip:1883" \ +EMQX_LISTENERS__SSL__DEFAULT__BIND="$ip:8883" \ +EMQX_LISTENERS__WS__DEFAULT__BIND="$ip:8083" \ +EMQX_LISTENERS__WSS__DEFAULT__BIND="$ip:8084" \ +EMQX_DASHBOARD__LISTENERS__HTTP__BIND="$ip:18083" \ +"$BOOT_SCRIPT" start +EOF +} + +start_node() { + local cmd + cmd="$(start_cmd "$1" "$2" | envsubst)" + echo "$cmd" + eval "$cmd" +} + +for id in "${CORE_IDS[@]}"; do + sudo ifconfig lo0 alias 127.0.0.$id up + start_node core "$id" & +done + +for id in "${REPLICANT_IDS[@]}"; do + sudo ifconfig lo0 alias 127.0.0.$id up + start_node replicant "$id" & +done From d781346efc3e173ead17e014ff5c88196aa98365 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 7 Jul 2023 20:25:49 +0800 Subject: [PATCH 78/92] fix: alias listeners.Type.Name.enabled as listeners.Type.Name.enable --- apps/emqx/src/emqx_listeners.erl | 8 ++++---- apps/emqx/src/emqx_schema.erl | 5 ++--- apps/emqx_management/src/emqx_mgmt_api_listeners.erl | 10 +++++----- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index aaee3b64e..689c0243a 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -368,7 +368,7 @@ console_print(_Fmt, _Args) -> ok. %% Start MQTT/TCP listener -spec do_start_listener(atom(), atom(), map()) -> {ok, pid() | {skipped, atom()}} | {error, term()}. -do_start_listener(_Type, _ListenerName, #{enabled := false}) -> +do_start_listener(_Type, _ListenerName, #{enable := false}) -> {ok, {skipped, listener_disabled}}; do_start_listener(Type, ListenerName, #{bind := ListenOn} = Opts) when Type == tcp; Type == ssl @@ -499,8 +499,8 @@ post_config_update([?ROOT_KEY, Type, Name], {update, _Request}, NewConf, OldConf post_config_update([?ROOT_KEY, Type, Name], ?MARK_DEL, _, OldConf = #{}, _AppEnvs) -> remove_listener(Type, Name, OldConf); post_config_update([?ROOT_KEY, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) -> - #{enabled := NewEnabled} = NewConf, - #{enabled := OldEnabled} = OldConf, + #{enable := NewEnabled} = NewConf, + #{enable := OldEnabled} = OldConf, case {NewEnabled, OldEnabled} of {true, true} -> ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf), @@ -810,7 +810,7 @@ has_enabled_listener_conf_by_type(Type) -> lists:any( fun({Id, LConf}) when is_map(LConf) -> {ok, #{type := Type0}} = parse_listener_id(Id), - Type =:= Type0 andalso maps:get(enabled, LConf, true) + Type =:= Type0 andalso maps:get(enable, LConf, true) end, list() ). diff --git a/apps/emqx/src/emqx_schema.erl b/apps/emqx/src/emqx_schema.erl index f552fae7f..ce1af66af 100644 --- a/apps/emqx/src/emqx_schema.erl +++ b/apps/emqx/src/emqx_schema.erl @@ -1750,13 +1750,12 @@ mqtt_listener(Bind) -> base_listener(Bind) -> [ - {"enabled", + {"enable", sc( boolean(), #{ default => true, - %% TODO(5.2): change field name to 'enable' and keep 'enabled' as an alias - aliases => [enable], + aliases => [enabled], desc => ?DESC(fields_listener_enabled) } )}, diff --git a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl index dd9013b16..86819e896 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_listeners.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_listeners.erl @@ -510,9 +510,9 @@ action_listeners_by_id(post, #{bindings := #{id := Id, action := Action}}) -> %%%============================================================================================== -enabled(start) -> #{<<"enabled">> => true}; -enabled(stop) -> #{<<"enabled">> => false}; -enabled(restart) -> #{<<"enabled">> => true}. +enabled(start) -> #{<<"enable">> => true}; +enabled(stop) -> #{<<"enable">> => false}; +enabled(restart) -> #{<<"enable">> => true}. err_msg(Atom) when is_atom(Atom) -> atom_to_binary(Atom); err_msg(Reason) -> list_to_binary(err_msg_str(Reason)). @@ -594,7 +594,7 @@ format_status(Key, Node, Listener, Acc) -> #{ <<"id">> := Id, <<"type">> := Type, - <<"enabled">> := Enabled, + <<"enable">> := Enable, <<"running">> := Running, <<"max_connections">> := MaxConnections, <<"current_connections">> := CurrentConnections, @@ -609,7 +609,7 @@ format_status(Key, Node, Listener, Acc) -> GroupKey => #{ name => Name, type => Type, - enable => Enabled, + enable => Enable, ids => [Id], acceptors => Acceptors, bind => iolist_to_binary(emqx_listeners:format_bind(Bind)), From c0ee47dc08b4797d091199139636fdcae0e54030 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 7 Jul 2023 21:26:59 +0800 Subject: [PATCH 79/92] chore: add changelog for 11226 pr --- apps/emqx/test/emqx_listeners_SUITE.erl | 2 +- apps/emqx_conf/test/emqx_conf_schema_tests.erl | 4 ++-- changes/ce/feat-11226.en.md | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changes/ce/feat-11226.en.md diff --git a/apps/emqx/test/emqx_listeners_SUITE.erl b/apps/emqx/test/emqx_listeners_SUITE.erl index fa0713cf0..b8d0c39f6 100644 --- a/apps/emqx/test/emqx_listeners_SUITE.erl +++ b/apps/emqx/test/emqx_listeners_SUITE.erl @@ -229,7 +229,7 @@ t_ssl_password_cert(Config) -> keyfile => filename:join(DataDir, "server-password.key") }, LConf = #{ - enabled => true, + enable => true, bind => {{127, 0, 0, 1}, Port}, mountpoint => <<>>, zone => default, diff --git a/apps/emqx_conf/test/emqx_conf_schema_tests.erl b/apps/emqx_conf/test/emqx_conf_schema_tests.erl index 855b8ff12..d5c864fab 100644 --- a/apps/emqx_conf/test/emqx_conf_schema_tests.erl +++ b/apps/emqx_conf/test/emqx_conf_schema_tests.erl @@ -363,14 +363,14 @@ listeners_test() -> ?assertMatch( #{ <<"bind">> := {{0, 0, 0, 0}, 1883}, - <<"enabled">> := true + <<"enable">> := true }, Tcp ), ?assertMatch( #{ <<"bind">> := {{0, 0, 0, 0}, 8083}, - <<"enabled">> := true, + <<"enable">> := true, <<"websocket">> := #{<<"mqtt_path">> := "/mqtt"} }, Ws diff --git a/changes/ce/feat-11226.en.md b/changes/ce/feat-11226.en.md new file mode 100644 index 000000000..0c43886bc --- /dev/null +++ b/changes/ce/feat-11226.en.md @@ -0,0 +1 @@ +Unify the listener switch to `enable`, while being compatible with the previous `enabled`. From 86d3984025fdcf96e34cc40e0c1721b812a089e4 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Fri, 7 Jul 2023 23:29:36 +0800 Subject: [PATCH 80/92] fix: mgmt api SUITE failed --- apps/emqx_management/test/emqx_mgmt_api_SUITE.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index a53ffc9c4..0e63b38ab 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -206,9 +206,9 @@ cluster_specs() -> {env, [{emqx, boot_modules, all}]}, {apps, []}, {conf, [ - {[listeners, ssl, default, enabled], false}, - {[listeners, ws, default, enabled], false}, - {[listeners, wss, default, enabled], false} + {[listeners, ssl, default, enable], false}, + {[listeners, ws, default, enable], false}, + {[listeners, wss, default, enable], false} ]} ], emqx_common_test_helpers:emqx_cluster( From d661b10355cac9efadf26ee2d4ee5158dd96cf21 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 10 Jul 2023 10:34:24 +0800 Subject: [PATCH 81/92] fix: emqx CI failed --- apps/emqx/test/emqx_common_test_helpers.erl | 2 +- .../test/emqx_eviction_agent_test_helpers.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index b004b139e..7f1fe4628 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -618,7 +618,7 @@ ensure_quic_listener(Name, UdpPort, ExtraSettings) -> "TLS_AES_128_GCM_SHA256", "TLS_CHACHA20_POLY1305_SHA256" ], - enabled => true, + enable => true, idle_timeout => 15000, ssl_options => #{ certfile => filename:join(code:lib_dir(emqx), "etc/certs/cert.pem"), diff --git a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl index 3953ec3e2..7425cb145 100644 --- a/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl +++ b/apps/emqx_eviction_agent/test/emqx_eviction_agent_test_helpers.erl @@ -84,7 +84,7 @@ start_cluster(NamesWithPorts, Apps, Env) -> {env, [{emqx, boot_modules, [broker, listeners]}] ++ Env}, {apps, Apps}, {conf, - [{[listeners, Proto, default, enabled], false} || Proto <- [ssl, ws, wss]] ++ + [{[listeners, Proto, default, enable], false} || Proto <- [ssl, ws, wss]] ++ [{[rpc, mode], async}]} ], Cluster = emqx_common_test_helpers:emqx_cluster( From c4ba558ee3b06b5fef05174a5021d89c8c4b1b07 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 10 Jul 2023 11:06:08 +0800 Subject: [PATCH 82/92] fix: unset config_loader in emqx's env when stop emqx app --- apps/emqx/src/emqx_app.erl | 16 +++++++++++++--- apps/emqx/src/emqx_listeners.erl | 2 +- apps/emqx_conf/src/emqx_conf_app.erl | 9 +-------- apps/emqx_conf/test/emqx_conf_app_SUITE.erl | 2 +- apps/emqx_machine/src/emqx_machine.app.src | 2 +- apps/emqx_machine/src/emqx_machine_boot.erl | 1 + 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index 038c93283..cb72986e7 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -25,7 +25,9 @@ get_description/0, get_release/0, set_config_loader/1, - get_config_loader/0 + get_config_loader/0, + unset_config_loaded/0, + init_load_done/0 ]). -include("logger.hrl"). @@ -54,14 +56,22 @@ prep_stop(_State) -> stop(_State) -> ok. +-define(CONFIG_LOADER, config_loader). +-define(DEFAULT_LOADER, emqx). %% @doc Call this function to make emqx boot without loading config, %% in case we want to delegate the config load to a higher level app %% which manages emqx app. set_config_loader(Module) when is_atom(Module) -> - application:set_env(emqx, config_loader, Module). + application:set_env(emqx, ?CONFIG_LOADER, Module). get_config_loader() -> - application:get_env(emqx, config_loader, emqx). + application:get_env(emqx, ?CONFIG_LOADER, ?DEFAULT_LOADER). + +unset_config_loaded() -> + application:unset_env(emqx, ?CONFIG_LOADER). + +init_load_done() -> + get_config_loader() =/= ?DEFAULT_LOADER. maybe_load_config() -> case get_config_loader() of diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index aaee3b64e..e59f834da 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -96,7 +96,7 @@ format_list(Listener) -> do_list_raw() -> %% GET /listeners from other nodes returns [] when init config is not loaded. - case emqx_app:get_config_loader() =/= emqx of + case emqx_app:init_load_done() of true -> Key = <<"listeners">>, Raw = emqx_config:get_raw([Key], #{}), diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 3c9af9393..1b241dfa5 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -22,9 +22,6 @@ -export([get_override_config_file/0]). -export([sync_data_from_node/0]). -%% Test purposes --export([init_load_done/0]). - -include_lib("emqx/include/logger.hrl"). -include("emqx_conf.hrl"). @@ -49,7 +46,7 @@ stop(_State) -> %% This function is named 'override' due to historical reasons. get_override_config_file() -> Node = node(), - case init_load_done() of + case emqx_app:init_load_done() of false -> {error, #{node => Node, msg => "init_conf_load_not_done"}}; true -> @@ -109,10 +106,6 @@ init_load(TnxId) -> }) end. -init_load_done() -> - % NOTE: Either us or some higher level (i.e. tests) code loaded config. - emqx_app:get_config_loader() =/= emqx. - init_conf() -> emqx_cluster_rpc:wait_for_cluster_rpc(), {ok, TnxId} = sync_cluster_conf(), diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index 2e3b40b87..2e5da3d44 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -215,7 +215,7 @@ assert_no_cluster_conf_copied([Node | Nodes], File) -> assert_config_load_done(Nodes) -> lists:foreach( fun(Node) -> - Done = rpc:call(Node, emqx_conf_app, init_load_done, []), + Done = rpc:call(Node, emqx_app, init_load_done, []), ?assert(Done, #{node => Node}) end, Nodes diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index b34b577e3..e81d4b53f 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.7"}, + {vsn, "0.2.8"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]}, diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index 1adec9c01..bc8c50663 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -61,6 +61,7 @@ start_autocluster() -> stop_apps() -> ?SLOG(notice, #{msg => "stopping_emqx_apps"}), _ = emqx_alarm_handler:unload(), + ok = emqx_app:unset_config_loaded(), lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())). %% Those port apps are terminated after the main apps From 802a50601aace7fa8b161d9e81553e6218a73a18 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 10 Jul 2023 12:03:36 +0800 Subject: [PATCH 83/92] chore: add comment for init_loader_done --- apps/emqx/src/emqx_listeners.erl | 2 ++ apps/emqx_conf/src/emqx_conf_app.erl | 6 ++++++ apps/emqx_machine/src/emqx_machine_boot.erl | 2 +- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/apps/emqx/src/emqx_listeners.erl b/apps/emqx/src/emqx_listeners.erl index e59f834da..b6c3a7b74 100644 --- a/apps/emqx/src/emqx_listeners.erl +++ b/apps/emqx/src/emqx_listeners.erl @@ -96,6 +96,8 @@ format_list(Listener) -> do_list_raw() -> %% GET /listeners from other nodes returns [] when init config is not loaded. + %% FIXME This is a workaround for the issue: + %% mria:running_nodes() sometime return node which not ready to accept rpc call. case emqx_app:init_load_done() of true -> Key = <<"listeners">>, diff --git a/apps/emqx_conf/src/emqx_conf_app.erl b/apps/emqx_conf/src/emqx_conf_app.erl index 1b241dfa5..0a486c829 100644 --- a/apps/emqx_conf/src/emqx_conf_app.erl +++ b/apps/emqx_conf/src/emqx_conf_app.erl @@ -21,6 +21,7 @@ -export([start/2, stop/1]). -export([get_override_config_file/0]). -export([sync_data_from_node/0]). +-export([unset_config_loaded/0]). -include_lib("emqx/include/logger.hrl"). -include("emqx_conf.hrl"). @@ -42,6 +43,11 @@ start(_StartType, _StartArgs) -> stop(_State) -> ok. +%% @doc emqx_conf relies on this flag to synchronize configuration between nodes. +%% Therefore, we must clean up this flag when emqx application is restarted by mria. +unset_config_loaded() -> + emqx_app:unset_config_loaded(). + %% Read the cluster config from the local node. %% This function is named 'override' due to historical reasons. get_override_config_file() -> diff --git a/apps/emqx_machine/src/emqx_machine_boot.erl b/apps/emqx_machine/src/emqx_machine_boot.erl index bc8c50663..b929f0d72 100644 --- a/apps/emqx_machine/src/emqx_machine_boot.erl +++ b/apps/emqx_machine/src/emqx_machine_boot.erl @@ -61,7 +61,7 @@ start_autocluster() -> stop_apps() -> ?SLOG(notice, #{msg => "stopping_emqx_apps"}), _ = emqx_alarm_handler:unload(), - ok = emqx_app:unset_config_loaded(), + ok = emqx_conf_app:unset_config_loaded(), lists:foreach(fun stop_one_app/1, lists:reverse(sorted_reboot_apps())). %% Those port apps are terminated after the main apps From e9f1d7f2bf6b192a17d2b1bba4a4e8bcf9113ae7 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Fri, 7 Jul 2023 17:42:28 +0800 Subject: [PATCH 84/92] fix: influxdb float serialization error --- .../src/emqx_bridge_influxdb_connector.erl | 6 ++++-- changes/ee/fix-11223.en.md | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 changes/ee/fix-11223.en.md diff --git a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl index 1fe5b4f78..be5ed6b1c 100644 --- a/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl +++ b/apps/emqx_bridge_influxdb/src/emqx_bridge_influxdb_connector.erl @@ -638,8 +638,10 @@ value_type([UInt, <<"u">>]) when is_integer(UInt) -> {uint, UInt}; -value_type([Float]) when is_float(Float) -> - Float; +%% write `1`, `1.0`, `-1.0` all as float +%% see also: https://docs.influxdata.com/influxdb/v2.7/reference/syntax/line-protocol/#float +value_type([Number]) when is_number(Number) -> + Number; value_type([<<"t">>]) -> 't'; value_type([<<"T">>]) -> diff --git a/changes/ee/fix-11223.en.md b/changes/ee/fix-11223.en.md new file mode 100644 index 000000000..6d97746be --- /dev/null +++ b/changes/ee/fix-11223.en.md @@ -0,0 +1,5 @@ +In InfluxDB bridging, if intend to write using the float data type but the placeholder represents the original value +as an integer without a decimal point during serialization, it will result in the failure of Influx Line Protocol serialization +and the inability to write to the InfluxDB bridge. + +See also: [InfluxDB v2.7 Line-Protocol](https://docs.influxdata.com/influxdb/v2.7/reference/syntax/line-protocol/#float) From 2570bc3090402136f3b6753dc18427c71ccc13a5 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 10 Jul 2023 08:41:11 +0200 Subject: [PATCH 85/92] fix(scripts): make shellcheck happy also use EMQX_NODE__ROLE instead of EMQX_NODE__DB_ROLE --- scripts/dev-cluster-host.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/dev-cluster-host.sh b/scripts/dev-cluster-host.sh index 9ce083707..330bf1896 100755 --- a/scripts/dev-cluster-host.sh +++ b/scripts/dev-cluster-host.sh @@ -86,14 +86,15 @@ start_cmd() { local id="$2" local ip="127.0.0.$id" local nodename="$role$id" - local nodehome="$(pwd)/tmp/$nodename" + local nodehome + nodehome="$(pwd)/tmp/$nodename" mkdir -p "${nodehome}/data" "${nodehome}/log" cat <<-EOF env DEBUG="${DEBUG:-0}" \ EMQX_NODE_NAME="$nodename@$ip" \ EMQX_CLUSTER__STATIC__SEEDS="$SEEDS" \ EMQX_CLUSTER__DISCOVERY_STRATEGY=static \ -EMQX_NODE__DB_ROLE="$role" \ +EMQX_NODE__ROLE="$role" \ EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL="${EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL:-debug}" \ EMQX_LOG__FILE_HANDLERS__DEFAULT__FILE="${nodehome}/log/emqx.log" \ EMQX_LOG_DIR="${nodehome}/log" \ @@ -115,11 +116,11 @@ start_node() { } for id in "${CORE_IDS[@]}"; do - sudo ifconfig lo0 alias 127.0.0.$id up + sudo ifconfig lo0 alias "127.0.0.$id" up start_node core "$id" & done for id in "${REPLICANT_IDS[@]}"; do - sudo ifconfig lo0 alias 127.0.0.$id up + sudo ifconfig lo0 alias "127.0.0.$id" up start_node replicant "$id" & done From 79903eeebc5f86e3bbeb0797cdb1336325cf5312 Mon Sep 17 00:00:00 2001 From: Florian Mueller Date: Fri, 14 Apr 2023 08:29:03 +0200 Subject: [PATCH 86/92] fix: correct spelling mistakes --- deploy/charts/emqx/templates/StatefulSet.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/charts/emqx/templates/StatefulSet.yaml b/deploy/charts/emqx/templates/StatefulSet.yaml index 3e9e39f2c..33efe70c7 100644 --- a/deploy/charts/emqx/templates/StatefulSet.yaml +++ b/deploy/charts/emqx/templates/StatefulSet.yaml @@ -107,14 +107,14 @@ spec: - name: wss containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }} - name: dashboard - containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTP__BIND | default 18083 }} {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }} - name: internalmqtt containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }} {{- end }} - {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }} + {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTPS__BIND) }} - name: dashboardtls - containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }} + containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTPS__BIND }} {{- end }} - name: ekka containerPort: 4370 @@ -147,14 +147,14 @@ spec: readinessProbe: httpGet: path: /status - port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} + port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTP__BIND | default 18083 }} initialDelaySeconds: 10 periodSeconds: 5 failureThreshold: 30 livenessProbe: httpGet: path: /status - port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }} + port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENERS__HTTP__BIND | default 18083 }} initialDelaySeconds: 60 periodSeconds: 30 failureThreshold: 10 From 0632d629cbd0448dbd4eb864e3787c39db9a8c9d Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 10 Jul 2023 16:32:29 +0800 Subject: [PATCH 87/92] fix: bad prometheus schema for headers --- apps/emqx_prometheus/src/emqx_prometheus.app.src | 2 +- apps/emqx_prometheus/src/emqx_prometheus_api.erl | 10 ++++------ apps/emqx_prometheus/src/emqx_prometheus_schema.erl | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 7252e4436..40a1bd498 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -2,7 +2,7 @@ {application, emqx_prometheus, [ {description, "Prometheus for EMQX"}, % strict semver, bump manually! - {vsn, "5.0.13"}, + {vsn, "5.0.14"}, {modules, []}, {registered, [emqx_prometheus_sup]}, {applications, [kernel, stdlib, prometheus, emqx, emqx_management]}, diff --git a/apps/emqx_prometheus/src/emqx_prometheus_api.erl b/apps/emqx_prometheus/src/emqx_prometheus_api.erl index c2bfaefc8..987386b61 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_api.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_api.erl @@ -130,10 +130,8 @@ prometheus_data_schema() -> description => <<"Get Prometheus Data. Note that support for JSON output is deprecated and will be removed in v5.2.">>, content => - #{ - 'application/json' => - #{schema => #{type => object}}, - 'text/plain' => - #{schema => #{type => string}} - } + [ + {'text/plain', #{schema => #{type => string}}}, + {'application/json', #{schema => #{type => object}}} + ] }. diff --git a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl index 3300e8b28..3884f7065 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus_schema.erl +++ b/apps/emqx_prometheus/src/emqx_prometheus_schema.erl @@ -59,7 +59,7 @@ fields("prometheus") -> ?HOCON( list({string(), string()}), #{ - default => [], + default => #{}, required => false, converter => fun ?MODULE:convert_headers/1, desc => ?DESC(headers) From 1b5d55cdbb3a965bbff6a2031b1483f3321b723c Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 10 Jul 2023 16:47:16 +0800 Subject: [PATCH 88/92] fix: delete bad link in emqx_prometheus --- apps/emqx_prometheus/src/emqx_prometheus.app.src | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/emqx_prometheus/src/emqx_prometheus.app.src b/apps/emqx_prometheus/src/emqx_prometheus.app.src index 40a1bd498..e6ee145ff 100644 --- a/apps/emqx_prometheus/src/emqx_prometheus.app.src +++ b/apps/emqx_prometheus/src/emqx_prometheus.app.src @@ -11,7 +11,6 @@ {licenses, ["Apache-2.0"]}, {maintainers, ["EMQX Team "]}, {links, [ - {"Homepage", "https://emqx.io/"}, - {"Github", "https://github.com/emqx/emqx-prometheus"} + {"Homepage", "https://emqx.io/"} ]} ]}. From dc32822f6f0eb85f57d6c302bca0e2ae2af4bef0 Mon Sep 17 00:00:00 2001 From: zhongwencool Date: Mon, 10 Jul 2023 16:51:31 +0800 Subject: [PATCH 89/92] chore: add changelog for 11237 --- changes/ce/fix-11237.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ce/fix-11237.en.md diff --git a/changes/ce/fix-11237.en.md b/changes/ce/fix-11237.en.md new file mode 100644 index 000000000..d6220040e --- /dev/null +++ b/changes/ce/fix-11237.en.md @@ -0,0 +1 @@ +The `headers` default value in /prometheus API should be a map instead of a list. From d795274f9666bee4f6598ace05b9c2c13d1e05ed Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 10 Jul 2023 13:53:13 +0200 Subject: [PATCH 90/92] ci: rerun apps version check on release --- .github/workflows/release.yaml | 18 ++++ scripts/rerun-apps-version-check.py | 122 ++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 scripts/rerun-apps-version-check.py diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 30de6f3b1..586142bbe 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -101,3 +101,21 @@ jobs: push "el/8" "packages/$PROFILE-$VERSION-el8-arm64.rpm" push "el/9" "packages/$PROFILE-$VERSION-el9-amd64.rpm" push "el/9" "packages/$PROFILE-$VERSION-el9-arm64.rpm" + + rerun-apps-version-check: + runs-on: ubuntu-22.04 + if: github.repository_owner == 'emqx' && github.event_name == 'release' + needs: + - upload + permissions: + pull-requests: read + checks: read + actions: write + steps: + - uses: actions/checkout@v3 + - name: trigger re-run of app versions check on open PRs + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 scripts/rerun-apps-version-check.py diff --git a/scripts/rerun-apps-version-check.py b/scripts/rerun-apps-version-check.py new file mode 100644 index 000000000..3b9fa1d3d --- /dev/null +++ b/scripts/rerun-apps-version-check.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# Usage: python3 rerun-apps-version-check.py -t -r +# +# Default repo is emqx/emqx +# +import requests +import http.client +import json +import os +import sys +import time +import math +import inspect +from optparse import OptionParser +from urllib.parse import urlparse, parse_qs +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +user_agent = sys.argv[0].split('/')[-1] + +def query(owner, repo): + return """ +query { + repository(owner: "%s", name: "%s") { + pullRequests(last: 25, states: OPEN) { + nodes { + url + commits(last: 1) { + nodes { + commit { + checkSuites(first: 17) { + nodes { + url + checkRuns(first: 1, filterBy: {checkName: "check_apps_version"}) { + nodes { + url + } + } + } + } + } + } + } + } + } + } +} + """ % (owner, repo) + + +def get_headers(token: str): + return {'Accept': 'application/vnd.github+json', + 'Authorization': f'Bearer {token}', + 'X-GitHub-Api-Version': '2022-11-28', + 'User-Agent': f'{user_agent}' + } + +def get_session(): + session = requests.Session() + + retries = Retry(total=10, + backoff_factor=1, # 1s + allowed_methods=None, + status_forcelist=[ 429, 500, 502, 503, 504 ]) # Retry on these status codes + + session.mount('https://', HTTPAdapter(max_retries=retries)) + + return session + +def get_check_suite_ids(token: str, repo: str): + session = get_session() + url = f'https://api.github.com/graphql' + [repo_owner, repo_repo] = repo.split('/') + r = session.post(url, headers=get_headers(token), json={'query': query(repo_owner, repo_repo)}) + if r.status_code == 200: + resp = r.json() + if not 'data' in resp: + print(f'Failed to fetch check runs: {r.status_code}\n{r.json()}') + sys.exit(1) + ids = [] + for pr in resp['data']['repository']['pullRequests']['nodes']: + if not pr['commits']['nodes']: + continue + if not pr['commits']['nodes'][0]['commit']['checkSuites']['nodes']: + continue + for node in pr['commits']['nodes'][0]['commit']['checkSuites']['nodes']: + if node['checkRuns']['nodes']: + id = node['checkRuns']['nodes'][0]['url'].rsplit('/', 1)[-1] + url_parsed = urlparse(node['url']) + params = parse_qs(url_parsed.query) + check_suite_id = params['check_suite_id'][0] + ids.extend([check_suite_id]) + return ids + else: + print(f'Failed to fetch check runs: {r.status_code}\n{r.text}') + sys.exit(1) + +def rerequest_check_suite(token: str, repo: str, check_suite_id: str): + session = get_session() + url = f'https://api.github.com/repos/{repo}/check-suites/{check_suite_id}/rerequest' + r = session.post(url, headers=get_headers(token)) + if r.status_code == 201: + print(f'Successfully triggered rerequest for check suite {check_suite_id}') + else: + print(f'Failed to trigger rerequest for check suite {check_suite_id}: {r.status_code}\n{r.text}') + +def main(): + parser = OptionParser() + parser.add_option("-r", "--repo", dest="repo", + help="github repo", default="emqx/emqx") + parser.add_option("-t", "--token", dest="gh_token", + help="github API token") + (options, args) = parser.parse_args() + + # Get github token from env var if provided, else use the one from command line. + # The token must be exported in the env from ${{ secrets.GITHUB_TOKEN }} in the workflow. + token = os.environ['GITHUB_TOKEN'] if 'GITHUB_TOKEN' in os.environ else options.gh_token + for id in get_check_suite_ids(token, options.repo): + rerequest_check_suite(token, options.repo, id) + +if __name__ == '__main__': + main() From e30bc6afa8b166d7d722f1c48165e68a0364fe84 Mon Sep 17 00:00:00 2001 From: JimMoen Date: Mon, 10 Jul 2023 18:50:38 +0800 Subject: [PATCH 91/92] test(influxdb): write raw as float for all number value --- .../test/emqx_bridge_influxdb_SUITE.erl | 77 +++++++++++++++---- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl index f97e5e977..3976d187a 100644 --- a/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl +++ b/apps/emqx_bridge_influxdb/test/emqx_bridge_influxdb_SUITE.erl @@ -454,24 +454,26 @@ query_by_clientid(ClientId, Config) -> {ok, DecodedCSV0} = erl_csv:decode(RawBody1, #{separator => <<$;>>}), DecodedCSV1 = [ [Field || Field <- Line, Field =/= <<>>] - || Line <- DecodedCSV0, - Line =/= [<<>>] + || Line <- DecodedCSV0, Line =/= [<<>>] ], - DecodedCSV2 = csv_lines_to_maps(DecodedCSV1, []), + DecodedCSV2 = csv_lines_to_maps(DecodedCSV1), index_by_field(DecodedCSV2). -decode_csv(RawBody) -> - Lines = - [ - binary:split(Line, [<<";">>], [global, trim_all]) - || Line <- binary:split(RawBody, [<<"\r\n">>], [global, trim_all]) - ], - csv_lines_to_maps(Lines, []). +csv_lines_to_maps([Title | Rest]) -> + csv_lines_to_maps(Rest, Title, _Acc = []); +csv_lines_to_maps([]) -> + []. -csv_lines_to_maps([Fields, Data | Rest], Acc) -> - Map = maps:from_list(lists:zip(Fields, Data)), - csv_lines_to_maps(Rest, [Map | Acc]); -csv_lines_to_maps(_Data, Acc) -> +csv_lines_to_maps([[<<"_result">> | _] = Data | RestData], Title, Acc) -> + Map = maps:from_list(lists:zip(Title, Data)), + csv_lines_to_maps(RestData, Title, [Map | Acc]); +%% ignore the csv title line +%% it's always like this: +%% [<<"result">>,<<"table">>,<<"_start">>,<<"_stop">>, +%% <<"_time">>,<<"_value">>,<<"_field">>,<<"_measurement">>, Measurement], +csv_lines_to_maps([[<<"result">> | _] = _Title | RestData], Title, Acc) -> + csv_lines_to_maps(RestData, Title, Acc); +csv_lines_to_maps([], _Title, Acc) -> lists:reverse(Acc). index_by_field(DecodedCSV) -> @@ -768,6 +770,53 @@ t_boolean_variants(Config) -> ), ok. +t_any_num_as_float(Config) -> + QueryMode = ?config(query_mode, Config), + Const = erlang:system_time(nanosecond), + ConstBin = integer_to_binary(Const), + TsStr = iolist_to_binary( + calendar:system_time_to_rfc3339(Const, [{unit, nanosecond}, {offset, "Z"}]) + ), + ?assertMatch( + {ok, _}, + create_bridge( + Config, + #{ + <<"write_syntax">> => + <<"mqtt,clientid=${clientid}", " ", + "float_no_dp=${payload.float_no_dp},float_dp=${payload.float_dp},bar=5i ", + ConstBin/binary>> + } + ) + ), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = #{ + %% no decimal point + float_no_dp => 123, + %% with decimal point + float_dp => 123.0 + }, + SentData = #{ + <<"clientid">> => ClientId, + <<"topic">> => atom_to_binary(?FUNCTION_NAME), + <<"payload">> => Payload, + <<"timestamp">> => erlang:system_time(millisecond) + }, + case QueryMode of + sync -> + ?assertMatch({ok, 204, _}, send_message(Config, SentData)), + ok; + async -> + ?assertMatch(ok, send_message(Config, SentData)), + ct:sleep(500) + end, + PersistedData = query_by_clientid(ClientId, Config), + Expected = #{float_no_dp => <<"123">>, float_dp => <<"123">>}, + assert_persisted_data(ClientId, Expected, PersistedData), + TimeReturned0 = maps:get(<<"_time">>, maps:get(<<"float_no_dp">>, PersistedData)), + TimeReturned = pad_zero(TimeReturned0), + ?assertEqual(TsStr, TimeReturned). + t_bad_timestamp(Config) -> InfluxDBType = ?config(influxdb_type, Config), InfluxDBName = ?config(influxdb_name, Config), From fb96c1e20de9b5ca4cfc4c6d72329ac8b3b07390 Mon Sep 17 00:00:00 2001 From: Ivan Dyachkov Date: Mon, 10 Jul 2023 17:24:00 +0200 Subject: [PATCH 92/92] chore: bump to e5.1.1-alpha.1 --- apps/emqx/include/emqx_release.hrl | 2 +- deploy/charts/emqx-enterprise/Chart.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 57a39816d..5446975d3 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -35,7 +35,7 @@ -define(EMQX_RELEASE_CE, "5.1.1"). %% Enterprise edition --define(EMQX_RELEASE_EE, "5.1.0"). +-define(EMQX_RELEASE_EE, "5.1.1-alpha.1"). %% The HTTP API version -define(EMQX_API_VERSION, "5.0"). diff --git a/deploy/charts/emqx-enterprise/Chart.yaml b/deploy/charts/emqx-enterprise/Chart.yaml index 821c19e8c..626436517 100644 --- a/deploy/charts/emqx-enterprise/Chart.yaml +++ b/deploy/charts/emqx-enterprise/Chart.yaml @@ -14,8 +14,8 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. -version: 5.1.0 +version: 5.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 5.1.0 +appVersion: 5.1.1