From 4d581d0b3091d4a4dff2e4618ea29867d2201aff Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Tue, 9 May 2023 14:43:22 +0800 Subject: [PATCH 01/21] feat: init commit --- apps/emqx_bridge_greptimedb/.gitignore | 19 ++ apps/emqx_bridge_greptimedb/LICENSE | 191 ++++++++++++++++++ apps/emqx_bridge_greptimedb/README.md | 19 ++ apps/emqx_bridge_greptimedb/rebar.config | 13 ++ .../src/emqx_bridge_greptimedb.app.src | 14 ++ .../src/emqx_bridge_greptimedb.erl | 3 + 6 files changed, 259 insertions(+) create mode 100644 apps/emqx_bridge_greptimedb/.gitignore create mode 100644 apps/emqx_bridge_greptimedb/LICENSE create mode 100644 apps/emqx_bridge_greptimedb/README.md create mode 100644 apps/emqx_bridge_greptimedb/rebar.config create mode 100644 apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src create mode 100644 apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl diff --git a/apps/emqx_bridge_greptimedb/.gitignore b/apps/emqx_bridge_greptimedb/.gitignore new file mode 100644 index 000000000..45f82dfcd --- /dev/null +++ b/apps/emqx_bridge_greptimedb/.gitignore @@ -0,0 +1,19 @@ +.rebar3 +_* +.eunit +*.o +*.beam +*.plt +*.swp +*.swo +.erlang.cookie +ebin + log +erl_crash.dump +.rebar +logs +_build +.idea +*.iml +rebar3.crashdump +*~ diff --git a/apps/emqx_bridge_greptimedb/LICENSE b/apps/emqx_bridge_greptimedb/LICENSE new file mode 100644 index 000000000..64d3c22a9 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/LICENSE @@ -0,0 +1,191 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2023, Dennis Zhuang . + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/apps/emqx_bridge_greptimedb/README.md b/apps/emqx_bridge_greptimedb/README.md new file mode 100644 index 000000000..13f26c348 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/README.md @@ -0,0 +1,19 @@ +# emqx_bridge_greptimedb +This application houses the GreptimeDB data integration to EMQX. +It provides the means to connect to GreptimeDB and publish messages to it. + +It implements connection management and interaction without the need for a + separate connector app, since it's not used for authentication and authorization + applications. + +## Docs + +For more information about GreptimeDB, please refer to [official + document](https://docs.greptime.com/). + +## Configurations + +TODO + +## License +[Apache License 2.0](./LICENSE) diff --git a/apps/emqx_bridge_greptimedb/rebar.config b/apps/emqx_bridge_greptimedb/rebar.config new file mode 100644 index 000000000..cb1385874 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/rebar.config @@ -0,0 +1,13 @@ +{erl_opts, [ + debug_info +]}. + +{deps, [ + {emqx, {path, "../../apps/emqx"}}, + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}}, + {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.0"}}} +]}. +{plugins, [rebar3_path_deps]}. +{project_plugins, [erlfmt]}. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src new file mode 100644 index 000000000..f63863d71 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -0,0 +1,14 @@ +{application, emqx_bridge_greptimedb, + [{description, "An OTP library"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, + [kernel, + stdlib + ]}, + {env,[]}, + {modules, []}, + + {licenses, ["Apache-2.0"]}, + {links, []} + ]}. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl new file mode 100644 index 000000000..ffb0e39c7 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -0,0 +1,3 @@ +-module(emqx_bridge_greptimedb). + +-export([]). From c5078980f3dd6d9324079813d1fc36505997d36f Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Tue, 9 May 2023 16:10:14 +0800 Subject: [PATCH 02/21] feat: adds the greptimedb bridge to emqx modules --- apps/emqx_bridge/src/emqx_bridge.erl | 3 ++- .../src/schema/emqx_bridge_enterprise.erl | 16 ++++++++++++++-- .../src/emqx_bridge_greptimedb_connector.erl | 1 + mix.exs | 1 + scripts/spellcheck/dicts/emqx.txt | 1 + 5 files changed, 19 insertions(+), 3 deletions(-) create mode 100644 apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index d5fc42ade..612481663 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -89,7 +89,8 @@ T == pulsar_producer; T == oracle; T == iotdb; - T == kinesis_producer + T == kinesis_producer; + T == greptimedb ). -define(ROOT_KEY, bridges). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index e4ef94c9e..7b9e1d4fa 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -49,7 +49,8 @@ api_schemas(Method) -> api_ref(emqx_bridge_oracle, <<"oracle">>, Method), api_ref(emqx_bridge_iotdb, <<"iotdb">>, Method), api_ref(emqx_bridge_rabbitmq, <<"rabbitmq">>, Method), - api_ref(emqx_bridge_kinesis, <<"kinesis_producer">>, Method ++ "_producer") + api_ref(emqx_bridge_kinesis, <<"kinesis_producer">>, Method ++ "_producer"), + api_ref(emqx_bridge_greptimedb, Method) ]. schema_modules() -> @@ -75,7 +76,8 @@ schema_modules() -> emqx_bridge_oracle, emqx_bridge_iotdb, emqx_bridge_rabbitmq, - emqx_bridge_kinesis + emqx_bridge_kinesis, + emqx_bridge_greptimedb ]. examples(Method) -> @@ -122,6 +124,8 @@ resource_type(oracle) -> emqx_oracle; resource_type(iotdb) -> emqx_bridge_iotdb_impl; resource_type(rabbitmq) -> emqx_bridge_rabbitmq_connector; resource_type(kinesis_producer) -> emqx_bridge_kinesis_impl_producer. +resource_type(rabbitmq) -> emqx_bridge_rabbitmq_connector. +resource_type(greptimedb) -> emqx_bridge_greptimedb_connector. fields(bridges) -> [ @@ -197,6 +201,14 @@ fields(bridges) -> desc => <<"Apache IoTDB Bridge Config">>, required => false } + )}, + {greptimedb, + mk( + hoconsc:map(name, ref(emqx_bridge_greptimedb, "config")), + #{ + desc => <<"GreptimeDB Bridge Config">>, + required => false + } )} ] ++ kafka_structs() ++ pulsar_structs() ++ gcp_pubsub_structs() ++ mongodb_structs() ++ influxdb_structs() ++ diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl new file mode 100644 index 000000000..8f7aa65e2 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -0,0 +1 @@ +-module(emqx_bridge_greptimedb_connector). diff --git a/mix.exs b/mix.exs index c6b685893..4d6cf700b 100644 --- a/mix.exs +++ b/mix.exs @@ -171,6 +171,7 @@ defmodule EMQXUmbrella.MixProject do :emqx_bridge_cassandra, :emqx_bridge_opents, :emqx_bridge_dynamo, + :emqx_bridge_greptimedb, :emqx_bridge_hstreamdb, :emqx_bridge_influxdb, :emqx_bridge_iotdb, diff --git a/scripts/spellcheck/dicts/emqx.txt b/scripts/spellcheck/dicts/emqx.txt index 953b0b762..b515a0010 100644 --- a/scripts/spellcheck/dicts/emqx.txt +++ b/scripts/spellcheck/dicts/emqx.txt @@ -29,6 +29,7 @@ EPMD ERL ETS FIXME +GreptimeDB GCM HMAC HOCON From 417e01749815d41562ec991df7413a339891ba0c Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Tue, 9 May 2023 21:34:30 +0800 Subject: [PATCH 03/21] feat: begin to impl connector --- .../src/emqx_bridge_greptimedb.app.src | 26 ++-- .../src/emqx_bridge_greptimedb_connector.erl | 113 ++++++++++++++++++ 2 files changed, 126 insertions(+), 13 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index f63863d71..14d655763 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -1,14 +1,14 @@ -{application, emqx_bridge_greptimedb, - [{description, "An OTP library"}, - {vsn, "0.1.0"}, - {registered, []}, - {applications, - [kernel, - stdlib - ]}, - {env,[]}, - {modules, []}, +{application, emqx_bridge_greptimedb, [ + {description, "An OTP library"}, + {vsn, "0.1.0"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []}, + {modules, []}, - {licenses, ["Apache-2.0"]}, - {links, []} - ]}. + {licenses, ["Apache-2.0"]}, + {links, []} +]}. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 8f7aa65e2..17c4d9a3c 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -1 +1,114 @@ -module(emqx_bridge_greptimedb_connector). +-include_lib("emqx/include/logger.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%% `emqx_resource' API +-export([ + callback_mode/0, + on_start/2, + on_stop/2, + on_get_status/2, + on_query/3, + on_query_async/4, + on_batch_query/3, + on_batch_query_async/4 +]). + +-define(GREPTIMEDB_DEFAULT_PORT, 4001). + +-define(GREPTIMEDB_HOST_OPTIONS, #{ + default_port => ?GREPTIMEDB_DEFAULT_PORT +}). + +%%------------------------------------------------------------------------------------- +%% `emqx_resource' API +%%------------------------------------------------------------------------------------- +callback_mode() -> async_if_possible. + +on_start(InstId, Config) -> + start_client(InstId, Config). + +on_stop(_InstId, #{client := Client}) -> + greptimedb:stop_client(Client). + +on_get_status(_InstId, _State) -> + %% FIXME + connected. + +on_query(_InstanceId, {send_message, _Message}, _State) -> + todo. + +on_query_async(_InstanceId, {send_message, _Message}, _ReplyFunAndArgs0, _State) -> + todo. + +on_batch_query( + _ResourceID, + _BatchReq, + _State +) -> + todo. + +on_batch_query_async( + _InstId, + _BatchData, + {_ReplyFun, _Args}, + _State +) -> + todo. + +%% internal functions + +start_client(InstId, Config) -> + ClientConfig = client_config(InstId, Config), + ?SLOG(info, #{ + msg => "starting GreptimeDB connector", + connector => InstId, + config => emqx_utils:redact(Config), + client_config => emqx_utils:redact(ClientConfig) + }), + try + case greptimedb:start_client(ClientConfig) of + {ok, Client} -> + {ok, #{client => Client}}; + {error, Reason} -> + ?tp(greptimedb_connector_start_failed, #{error => Reason}), + ?SLOG(warning, #{ + msg => "failed_to_start_greptimedb_connector", + connector => InstId, + reason => Reason + }), + {error, Reason} + end + catch + E:R:S -> + ?tp(greptimedb_connector_start_exception, #{error => {E, R}}), + ?SLOG(warning, #{ + msg => "start greptimedb connector error", + connector => InstId, + error => E, + reason => R, + stack => S + }), + {error, R} + end. + +client_config( + InstId, + _Config = #{ + server := Server + } +) -> + #{hostname := Host, port := Port} = emqx_schema:parse_server(Server, ?GREPTIMEDB_HOST_OPTIONS), + [ + {endpoints, [{http, str(Host), Port}]}, + {pool_size, erlang:system_info(schedulers)}, + {pool, InstId}, + {pool_type, random} + ]. + +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. From 6d9944a8e8dd37b59265a70b19a37ee52772d092 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Tue, 4 Jul 2023 15:19:14 +0800 Subject: [PATCH 04/21] feat: update greptimedb dependencies --- apps/emqx_bridge_greptimedb/rebar.config | 2 +- .../src/emqx_bridge_greptimedb.app.src | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/rebar.config b/apps/emqx_bridge_greptimedb/rebar.config index cb1385874..cbde4660f 100644 --- a/apps/emqx_bridge_greptimedb/rebar.config +++ b/apps/emqx_bridge_greptimedb/rebar.config @@ -7,7 +7,7 @@ {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}}, - {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.0"}}} + {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.1"}}} ]}. {plugins, [rebar3_path_deps]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index 14d655763..f0a07bc28 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -1,14 +1,14 @@ {application, emqx_bridge_greptimedb, [ - {description, "An OTP library"}, + {description, "EMQX GreptimeDB Bridge"}, {vsn, "0.1.0"}, {registered, []}, {applications, [ kernel, - stdlib + stdlib, + greptimedb ]}, {env, []}, {modules, []}, - {licenses, ["Apache-2.0"]}, {links, []} ]}. From 91ebd90442d9f77ee89f6ac1b0d4f50532feb754 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Thu, 6 Jul 2023 12:12:08 +0800 Subject: [PATCH 05/21] fix: batch write --- apps/emqx_bridge_greptimedb/rebar.config | 9 +- .../src/emqx_bridge_greptimedb.erl | 297 ++++++++- .../src/emqx_bridge_greptimedb_connector.erl | 601 ++++++++++++++++-- .../test/emqx_bridge_greptimedb_tests.erl | 348 ++++++++++ 4 files changed, 1202 insertions(+), 53 deletions(-) create mode 100644 apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_tests.erl diff --git a/apps/emqx_bridge_greptimedb/rebar.config b/apps/emqx_bridge_greptimedb/rebar.config index cbde4660f..952281286 100644 --- a/apps/emqx_bridge_greptimedb/rebar.config +++ b/apps/emqx_bridge_greptimedb/rebar.config @@ -3,11 +3,10 @@ ]}. {deps, [ - {emqx, {path, "../../apps/emqx"}}, - {emqx_connector, {path, "../../apps/emqx_connector"}}, - {emqx_resource, {path, "../../apps/emqx_resource"}}, - {emqx_bridge, {path, "../../apps/emqx_bridge"}}, - {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.1"}}} + {emqx_connector, {path, "../../apps/emqx_connector"}}, + {emqx_resource, {path, "../../apps/emqx_resource"}}, + {emqx_bridge, {path, "../../apps/emqx_bridge"}}, + {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.1"}}} ]}. {plugins, [rebar3_path_deps]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl index ffb0e39c7..f37ddf320 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -1,3 +1,298 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- -module(emqx_bridge_greptimedb). --export([]). +-include_lib("emqx/include/logger.hrl"). +-include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("typerefl/include/types.hrl"). +-include_lib("hocon/include/hoconsc.hrl"). + +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-export([ + conn_bridge_examples/1 +]). + +-export([ + namespace/0, + roots/0, + fields/1, + desc/1 +]). + +-type write_syntax() :: list(). +-reflect_type([write_syntax/0]). +-typerefl_from_string({write_syntax/0, ?MODULE, to_influx_lines}). +-export([to_influx_lines/1]). + +%% ------------------------------------------------------------------------------------------------- +%% api + +conn_bridge_examples(Method) -> + [ + #{ + <<"greptimedb_grpc_v1">> => #{ + summary => <<"Greptimedb HTTP API V2 Bridge">>, + value => values("greptimedb_grpc_v1", Method) + } + } + ]. + +values(Protocol, get) -> + values(Protocol, post); +values("greptimedb_grpc_v1", post) -> + SupportUint = <<"uint_value=${payload.uint_key}u,">>, + TypeOpts = #{ + bucket => <<"example_bucket">>, + org => <<"examlpe_org">>, + token => <<"example_token">>, + server => <<"127.0.0.1:4000">> + }, + values(common, "greptimedb_grpc_v1", SupportUint, TypeOpts); +values(Protocol, put) -> + values(Protocol, post). + +values(common, Protocol, SupportUint, TypeOpts) -> + CommonConfigs = #{ + type => list_to_atom(Protocol), + name => <<"demo">>, + enable => true, + local_topic => <<"local/topic/#">>, + write_syntax => + <<"${topic},clientid=${clientid}", " ", "payload=${payload},", + "${clientid}_int_value=${payload.int_key}i,", SupportUint/binary, + "bool=${payload.bool}">>, + precision => ms, + resource_opts => #{ + batch_size => 100, + batch_time => <<"20ms">> + }, + server => <<"127.0.0.1:4000">>, + ssl => #{enable => false} + }, + maps:merge(TypeOpts, CommonConfigs). + +%% ------------------------------------------------------------------------------------------------- +%% Hocon Schema Definitions +namespace() -> "bridge_greptimedb". + +roots() -> []. + +fields("post_grpc_v1") -> + method_fields(post, greptimedb_grpc_v1); +fields("put_grpc_v1") -> + method_fields(put, greptimedb_grpc_v1); +fields("get_grpc_v1") -> + method_fields(get, greptimedb_grpc_v1); +fields(Type) when + Type == greptimedb_grpc_v1 +-> + greptimedb_bridge_common_fields() ++ + connector_fields(Type). + +method_fields(post, ConnectorType) -> + greptimedb_bridge_common_fields() ++ + connector_fields(ConnectorType) ++ + type_name_fields(ConnectorType); +method_fields(get, ConnectorType) -> + greptimedb_bridge_common_fields() ++ + connector_fields(ConnectorType) ++ + type_name_fields(ConnectorType) ++ + emqx_bridge_schema:status_fields(); +method_fields(put, ConnectorType) -> + greptimedb_bridge_common_fields() ++ + connector_fields(ConnectorType). + +greptimedb_bridge_common_fields() -> + emqx_bridge_schema:common_bridge_fields() ++ + [ + {write_syntax, fun write_syntax/1} + ] ++ + emqx_resource_schema:fields("resource_opts"). + +connector_fields(Type) -> + emqx_bridge_greptimedb_connector:fields(Type). + +type_name_fields(Type) -> + [ + {type, mk(Type, #{required => true, desc => ?DESC("desc_type")})}, + {name, mk(binary(), #{required => true, desc => ?DESC("desc_name")})} + ]. + +desc("config") -> + ?DESC("desc_config"); +desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> + ["Configuration for Greptimedb using `", string:to_upper(Method), "` method."]; +desc(greptimedb_grpc_v1) -> + ?DESC(emqx_bridge_greptimedb_connector, "greptimedb_grpc_v1"); +desc(_) -> + undefined. + +write_syntax(type) -> + ?MODULE:write_syntax(); +write_syntax(required) -> + true; +write_syntax(validator) -> + [?NOT_EMPTY("the value of the field 'write_syntax' cannot be empty")]; +write_syntax(converter) -> + fun to_influx_lines/1; +write_syntax(desc) -> + ?DESC("write_syntax"); +write_syntax(format) -> + <<"sql">>; +write_syntax(_) -> + undefined. + +to_influx_lines(RawLines) -> + try + influx_lines(str(RawLines), []) + catch + _:Reason:Stacktrace -> + Msg = lists:flatten( + io_lib:format("Unable to parse Greptimedb line protocol: ~p", [RawLines]) + ), + ?SLOG(error, #{msg => Msg, error_reason => Reason, stacktrace => Stacktrace}), + throw(Msg) + end. + +-define(MEASUREMENT_ESC_CHARS, [$,, $\s]). +-define(TAG_FIELD_KEY_ESC_CHARS, [$,, $=, $\s]). +-define(FIELD_VAL_ESC_CHARS, [$", $\\]). +% Common separator for both tags and fields +-define(SEP, $\s). +-define(MEASUREMENT_TAG_SEP, $,). +-define(KEY_SEP, $=). +-define(VAL_SEP, $,). +-define(NON_EMPTY, [_ | _]). + +influx_lines([] = _RawLines, Acc) -> + ?NON_EMPTY = lists:reverse(Acc); +influx_lines(RawLines, Acc) -> + {Acc1, RawLines1} = influx_line(string:trim(RawLines, leading, "\s\n"), Acc), + influx_lines(RawLines1, Acc1). + +influx_line([], Acc) -> + {Acc, []}; +influx_line(Line, Acc) -> + {?NON_EMPTY = Measurement, Line1} = measurement(Line), + {Tags, Line2} = tags(Line1), + {?NON_EMPTY = Fields, Line3} = influx_fields(Line2), + {Timestamp, Line4} = timestamp(Line3), + { + [ + #{ + measurement => Measurement, + tags => Tags, + fields => Fields, + timestamp => Timestamp + } + | Acc + ], + Line4 + }. + +measurement(Line) -> + unescape(?MEASUREMENT_ESC_CHARS, [?MEASUREMENT_TAG_SEP, ?SEP], Line, []). + +tags([?MEASUREMENT_TAG_SEP | Line]) -> + tags1(Line, []); +tags(Line) -> + {[], Line}. + +%% Empty line is invalid as fields are required after tags, +%% need to break recursion here and fail later on parsing fields +tags1([] = Line, Acc) -> + {lists:reverse(Acc), Line}; +%% Matching non empty Acc treats lines like "m, field=field_val" invalid +tags1([?SEP | _] = Line, ?NON_EMPTY = Acc) -> + {lists:reverse(Acc), Line}; +tags1(Line, Acc) -> + {Tag, Line1} = tag(Line), + tags1(Line1, [Tag | Acc]). + +tag(Line) -> + {?NON_EMPTY = Key, Line1} = key(Line), + {?NON_EMPTY = Val, Line2} = tag_val(Line1), + {{Key, Val}, Line2}. + +tag_val(Line) -> + {Val, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?VAL_SEP, ?SEP], Line, []), + {Val, strip_l(Line1, ?VAL_SEP)}. + +influx_fields([?SEP | Line]) -> + fields1(string:trim(Line, leading, "\s"), []). + +%% Timestamp is optional, so fields may be at the very end of the line +fields1([Ch | _] = Line, Acc) when Ch =:= ?SEP; Ch =:= $\n -> + {lists:reverse(Acc), Line}; +fields1([] = Line, Acc) -> + {lists:reverse(Acc), Line}; +fields1(Line, Acc) -> + {Field, Line1} = field(Line), + fields1(Line1, [Field | Acc]). + +field(Line) -> + {?NON_EMPTY = Key, Line1} = key(Line), + {Val, Line2} = field_val(Line1), + {{Key, Val}, Line2}. + +field_val([$" | Line]) -> + {Val, [$" | Line1]} = unescape(?FIELD_VAL_ESC_CHARS, [$"], Line, []), + %% Quoted val can be empty + {Val, strip_l(Line1, ?VAL_SEP)}; +field_val(Line) -> + %% Unquoted value should not be un-escaped according to Greptimedb protocol, + %% as it can only hold float, integer, uinteger or boolean value. + %% However, as templates are possible, un-escaping is applied here, + %% which also helps to detect some invalid lines, e.g.: "m,tag=1 field= ${timestamp}" + {Val, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?VAL_SEP, ?SEP, $\n], Line, []), + {?NON_EMPTY = Val, strip_l(Line1, ?VAL_SEP)}. + +timestamp([?SEP | Line]) -> + Line1 = string:trim(Line, leading, "\s"), + %% Similarly to unquoted field value, un-escape a timestamp to validate and handle + %% potentially escaped characters in a template + {T, Line2} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?SEP, $\n], Line1, []), + {timestamp1(T), Line2}; +timestamp(Line) -> + {undefined, Line}. + +timestamp1(?NON_EMPTY = Ts) -> Ts; +timestamp1(_Ts) -> undefined. + +%% Common for both tag and field keys +key(Line) -> + {Key, Line1} = unescape(?TAG_FIELD_KEY_ESC_CHARS, [?KEY_SEP], Line, []), + {Key, strip_l(Line1, ?KEY_SEP)}. + +%% Only strip a character between pairs, don't strip it(and let it fail) +%% if the char to be stripped is at the end, e.g.: m,tag=val, field=val +strip_l([Ch, Ch1 | Str], Ch) when Ch1 =/= ?SEP -> + [Ch1 | Str]; +strip_l(Str, _Ch) -> + Str. + +unescape(EscapeChars, SepChars, [$\\, Char | T], Acc) -> + ShouldEscapeBackslash = lists:member($\\, EscapeChars), + Acc1 = + case lists:member(Char, EscapeChars) of + true -> [Char | Acc]; + false when not ShouldEscapeBackslash -> [Char, $\\ | Acc] + end, + unescape(EscapeChars, SepChars, T, Acc1); +unescape(EscapeChars, SepChars, [Char | T] = L, Acc) -> + IsEscapeChar = lists:member(Char, EscapeChars), + case lists:member(Char, SepChars) of + true -> {lists:reverse(Acc), L}; + false when not IsEscapeChar -> unescape(EscapeChars, SepChars, T, [Char | Acc]) + end; +unescape(_EscapeChars, _SepChars, [] = L, Acc) -> + {lists:reverse(Acc), L}. + +str(A) when is_atom(A) -> + atom_to_list(A); +str(B) when is_binary(B) -> + binary_to_list(B); +str(S) when is_list(S) -> + S. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 17c4d9a3c..a02df09c5 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -1,84 +1,179 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- -module(emqx_bridge_greptimedb_connector). + +-include_lib("emqx_connector/include/emqx_connector.hrl"). + +-include_lib("hocon/include/hoconsc.hrl"). +-include_lib("typerefl/include/types.hrl"). -include_lib("emqx/include/logger.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). -%% `emqx_resource' API +-import(hoconsc, [mk/2, enum/1, ref/2]). + +-behaviour(emqx_resource). + +%% callbacks of behaviour emqx_resource -export([ callback_mode/0, on_start/2, on_stop/2, - on_get_status/2, on_query/3, - on_query_async/4, on_batch_query/3, - on_batch_query_async/4 + on_get_status/2 ]). --define(GREPTIMEDB_DEFAULT_PORT, 4001). +-export([ + roots/0, + namespace/0, + fields/1, + desc/1 +]). + +%% only for test +-export([is_unrecoverable_error/1]). + +-type ts_precision() :: ns | us | ms | s. + +%% Allocatable resources +-define(greptime_client, greptime_client). + +-define(GREPTIMEDB_DEFAULT_PORT, 4000). + +-define(DEFAULT_DB, <<"public">>). -define(GREPTIMEDB_HOST_OPTIONS, #{ default_port => ?GREPTIMEDB_DEFAULT_PORT }). -%%------------------------------------------------------------------------------------- -%% `emqx_resource' API -%%------------------------------------------------------------------------------------- -callback_mode() -> async_if_possible. +-define(DEFAULT_TIMESTAMP_TMPL, "${timestamp}"). + +%% ------------------------------------------------------------------------------------------------- +%% resource callback +callback_mode() -> always_sync. on_start(InstId, Config) -> + %% InstID as pool would be handled by greptimedb client + %% so there is no need to allocate pool_name here + %% See: greptimedb:start_client/1 start_client(InstId, Config). -on_stop(_InstId, #{client := Client}) -> - greptimedb:stop_client(Client). +on_stop(InstId, _State) -> + case emqx_resource:get_allocated_resources(InstId) of + #{?greptime_client := Client} -> + greptimedb:stop_client(Client); + _ -> + ok + end. -on_get_status(_InstId, _State) -> - %% FIXME - connected. +on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, client := Client}) -> + case data_to_points(Data, SyntaxLines) of + {ok, Points} -> + ?tp( + greptimedb_connector_send_query, + #{points => Points, batch => false, mode => sync} + ), + do_query(InstId, Client, Points); + {error, ErrorPoints} -> + ?tp( + greptimedb_connector_send_query_error, + #{batch => false, mode => sync, error => ErrorPoints} + ), + log_error_points(InstId, ErrorPoints), + {error, {unrecoverable_error, ErrorPoints}} + end. -on_query(_InstanceId, {send_message, _Message}, _State) -> - todo. +%% Once a Batched Data trans to points failed. +%% This batch query failed +on_batch_query(InstId, BatchData, _State = #{write_syntax := SyntaxLines, client := Client}) -> + case parse_batch_data(InstId, BatchData, SyntaxLines) of + {ok, Points} -> + ?tp( + greptimedb_connector_send_query, + #{points => Points, batch => true, mode => sync} + ), + do_query(InstId, Client, Points); + {error, Reason} -> + ?tp( + greptimedb_connector_send_query_error, + #{batch => true, mode => sync, error => Reason} + ), + {error, {unrecoverable_error, Reason}} + end. -on_query_async(_InstanceId, {send_message, _Message}, _ReplyFunAndArgs0, _State) -> - todo. +on_get_status(_InstId, #{client := Client}) -> + case greptimedb:is_alive(Client) of + true -> + connected; + false -> + disconnected + end. -on_batch_query( - _ResourceID, - _BatchReq, - _State -) -> - todo. +%% ------------------------------------------------------------------------------------------------- +%% schema +namespace() -> connector_greptimedb. -on_batch_query_async( - _InstId, - _BatchData, - {_ReplyFun, _Args}, - _State -) -> - todo. +roots() -> + [ + {config, #{ + type => hoconsc:union( + [ + hoconsc:ref(?MODULE, greptimedb_grpc_v1) + ] + ) + }} + ]. +fields(common) -> + [ + {server, server()}, + {precision, + %% The greptimedb only supports these 4 precision: + %% See "https://github.com/influxdata/greptimedb/blob/ + %% 6b607288439a991261307518913eb6d4e280e0a7/models/points.go#L487" for + %% more information. + mk(enum([ns, us, ms, s]), #{ + required => false, default => ms, desc => ?DESC("precision") + })} + ]; +fields(greptimedb_grpc_v1) -> + fields(common) ++ + [ + {dbname, mk(binary(), #{required => true, desc => ?DESC("dbname")})} + ] ++ emqx_connector_schema_lib:ssl_fields(). + +server() -> + Meta = #{ + required => false, + default => <<"127.0.0.1:4000">>, + desc => ?DESC("server"), + converter => fun convert_server/2 + }, + emqx_schema:servers_sc(Meta, ?GREPTIMEDB_HOST_OPTIONS). + +desc(common) -> + ?DESC("common"); +desc(greptimedb_grpc_v1) -> + ?DESC("greptimedb_grpc_v1"). + +%% ------------------------------------------------------------------------------------------------- %% internal functions start_client(InstId, Config) -> ClientConfig = client_config(InstId, Config), ?SLOG(info, #{ - msg => "starting GreptimeDB connector", + msg => "starting greptimedb connector", connector => InstId, config => emqx_utils:redact(Config), client_config => emqx_utils:redact(ClientConfig) }), - try - case greptimedb:start_client(ClientConfig) of - {ok, Client} -> - {ok, #{client => Client}}; - {error, Reason} -> - ?tp(greptimedb_connector_start_failed, #{error => Reason}), - ?SLOG(warning, #{ - msg => "failed_to_start_greptimedb_connector", - connector => InstId, - reason => Reason - }), - {error, Reason} - end + try do_start_client(InstId, ClientConfig, Config) of + Res = {ok, #{client := Client}} -> + ok = emqx_resource:allocate_resource(InstId, ?greptime_client, Client), + Res; + {error, Reason} -> + {error, Reason} catch E:R:S -> ?tp(greptimedb_connector_start_exception, #{error => {E, R}}), @@ -92,9 +187,64 @@ start_client(InstId, Config) -> {error, R} end. +do_start_client( + InstId, + ClientConfig, + Config = #{write_syntax := Lines} +) -> + Precision = maps:get(precision, Config, ms), + case greptimedb:start_client(ClientConfig) of + {ok, Client} -> + case greptimedb:is_alive(Client, true) of + true -> + State = #{ + client => Client, + dbname => proplists:get_value(dbname, ClientConfig, ?DEFAULT_DB), + write_syntax => to_config(Lines, Precision) + }, + ?SLOG(info, #{ + msg => "starting greptimedb connector success", + connector => InstId, + client => redact_auth(Client), + state => redact_auth(State) + }), + {ok, State}; + {false, Reason} -> + ?tp(greptimedb_connector_start_failed, #{ + error => greptimedb_client_not_alive, reason => Reason + }), + ?SLOG(warning, #{ + msg => "failed_to_start_greptimedb_connector", + connector => InstId, + client => redact_auth(Client), + reason => Reason + }), + %% no leak + _ = greptimedb:stop_client(Client), + {error, greptimedb_client_not_alive} + end; + {error, {already_started, Client0}} -> + ?tp(greptimedb_connector_start_already_started, #{}), + ?SLOG(info, #{ + msg => "restarting greptimedb connector, found already started client", + connector => InstId, + old_client => redact_auth(Client0) + }), + _ = greptimedb:stop_client(Client0), + do_start_client(InstId, ClientConfig, Config); + {error, Reason} -> + ?tp(greptimedb_connector_start_failed, #{error => Reason}), + ?SLOG(warning, #{ + msg => "failed_to_start_greptimedb_connector", + connector => InstId, + reason => Reason + }), + {error, Reason} + end. + client_config( InstId, - _Config = #{ + Config = #{ server := Server } ) -> @@ -103,12 +253,369 @@ client_config( {endpoints, [{http, str(Host), Port}]}, {pool_size, erlang:system_info(schedulers)}, {pool, InstId}, - {pool_type, random} + {pool_type, random}, + {timeunit, maps:get(precision, Config, ms)} + ] ++ protocol_config(Config). + +protocol_config( + #{ + dbname := DbName, + ssl := SSL + } = Config +) -> + [ + {dbname, DbName} + ] ++ auth(Config) ++ + ssl_config(SSL). + +ssl_config(#{enable := false}) -> + [ + {https_enabled, false} + ]; +ssl_config(SSL = #{enable := true}) -> + [ + {https_enabled, true}, + {transport, ssl}, + {transport_opts, emqx_tls_lib:to_client_opts(SSL)} ]. +auth(#{username := Username, password := Password}) -> + [ + {auth, {basic, #{username => Username, password => Password}}} + ]; +auth(_) -> + []. + +redact_auth(Term) -> + emqx_utils:redact(Term, fun is_auth_key/1). + +is_auth_key(Key) when is_binary(Key) -> + string:equal("authorization", Key, true); +is_auth_key(_) -> + false. + +%% ------------------------------------------------------------------------------------------------- +%% Query +do_query(InstId, Client, Points) -> + case greptimedb:write_batch(Client, Points) of + {ok, _} -> + ?SLOG(debug, #{ + msg => "greptimedb write point success", + connector => InstId, + points => Points + }); + {error, {401, _, _}} -> + ?tp(greptimedb_connector_do_query_failure, #{error => <<"authorization failure">>}), + ?SLOG(error, #{ + msg => "greptimedb_authorization_failed", + client => redact_auth(Client), + connector => InstId + }), + {error, {unrecoverable_error, <<"authorization failure">>}}; + {error, Reason} = Err -> + ?tp(greptimedb_connector_do_query_failure, #{error => Reason}), + ?SLOG(error, #{ + msg => "greptimedb write point failed", + connector => InstId, + reason => Reason + }), + case is_unrecoverable_error(Err) of + true -> + {error, {unrecoverable_error, Reason}}; + false -> + {error, {recoverable_error, Reason}} + end + end. + +%% ------------------------------------------------------------------------------------------------- +%% Tags & Fields Config Trans + +to_config(Lines, Precision) -> + to_config(Lines, [], Precision). + +to_config([], Acc, _Precision) -> + lists:reverse(Acc); +to_config([Item0 | Rest], Acc, Precision) -> + Ts0 = maps:get(timestamp, Item0, undefined), + {Ts, FromPrecision, ToPrecision} = preproc_tmpl_timestamp(Ts0, Precision), + Item = #{ + measurement => emqx_placeholder:preproc_tmpl(maps:get(measurement, Item0)), + timestamp => Ts, + precision => {FromPrecision, ToPrecision}, + tags => to_kv_config(maps:get(tags, Item0)), + fields => to_kv_config(maps:get(fields, Item0)) + }, + to_config(Rest, [Item | Acc], Precision). + +%% pre-process the timestamp template +%% returns a tuple of three elements: +%% 1. The timestamp template itself. +%% 2. The source timestamp precision (ms if the template ${timestamp} is used). +%% 3. The target timestamp precision (configured for the client). +preproc_tmpl_timestamp(undefined, Precision) -> + %% not configured, we default it to the message timestamp + preproc_tmpl_timestamp(?DEFAULT_TIMESTAMP_TMPL, Precision); +preproc_tmpl_timestamp(Ts, Precision) when is_integer(Ts) -> + %% a const value is used which is very much unusual, but we have to add a special handling + {Ts, Precision, Precision}; +preproc_tmpl_timestamp(Ts, Precision) when is_list(Ts) -> + preproc_tmpl_timestamp(iolist_to_binary(Ts), Precision); +preproc_tmpl_timestamp(<> = Ts, Precision) -> + {emqx_placeholder:preproc_tmpl(Ts), ms, Precision}; +preproc_tmpl_timestamp(Ts, Precision) when is_binary(Ts) -> + %% a placehold is in use. e.g. ${payload.my_timestamp} + %% we can only hope it the value will be of the same precision in the configs + {emqx_placeholder:preproc_tmpl(Ts), Precision, Precision}. + +to_kv_config(KVfields) -> + maps:fold(fun to_maps_config/3, #{}, proplists:to_map(KVfields)). + +to_maps_config(K, V, Res) -> + NK = emqx_placeholder:preproc_tmpl(bin(K)), + NV = emqx_placeholder:preproc_tmpl(bin(V)), + Res#{NK => NV}. + +%% ------------------------------------------------------------------------------------------------- +%% Tags & Fields Data Trans +parse_batch_data(InstId, BatchData, SyntaxLines) -> + {Points, Errors} = lists:foldl( + fun({send_message, Data}, {ListOfPoints, ErrAccIn}) -> + case data_to_points(Data, SyntaxLines) of + {ok, Points} -> + {[Points | ListOfPoints], ErrAccIn}; + {error, ErrorPoints} -> + log_error_points(InstId, ErrorPoints), + {ListOfPoints, ErrAccIn + 1} + end + end, + {[], 0}, + BatchData + ), + case Errors of + 0 -> + {ok, lists:flatten(Points)}; + _ -> + ?SLOG(error, #{ + msg => io_lib:format("Greptimedb trans point failed, count: ~p", [Errors]), + connector => InstId, + reason => points_trans_failed + }), + {error, points_trans_failed} + end. + +-spec data_to_points(map(), [ + #{ + fields := [{binary(), binary()}], + measurement := binary(), + tags := [{binary(), binary()}], + timestamp := emqx_placeholder:tmpl_token() | integer(), + precision := {From :: ts_precision(), To :: ts_precision()} + } +]) -> {ok, [map()]} | {error, term()}. +data_to_points(Data, SyntaxLines) -> + lines_to_points(Data, SyntaxLines, [], []). + +%% When converting multiple rows data into Greptimedb Line Protocol, they are considered to be strongly correlated. +%% And once a row fails to convert, all of them are considered to have failed. +lines_to_points(_, [], Points, ErrorPoints) -> + case ErrorPoints of + [] -> + {ok, Points}; + _ -> + %% ignore trans succeeded points + {error, ErrorPoints} + end; +lines_to_points(Data, [#{timestamp := Ts} = Item | Rest], ResultPointsAcc, ErrorPointsAcc) when + is_list(Ts) +-> + TransOptions = #{return => rawlist, var_trans => fun data_filter/1}, + case parse_timestamp(emqx_placeholder:proc_tmpl(Ts, Data, TransOptions)) of + {ok, TsInt} -> + Item1 = Item#{timestamp => TsInt}, + continue_lines_to_points(Data, Item1, Rest, ResultPointsAcc, ErrorPointsAcc); + {error, BadTs} -> + lines_to_points(Data, Rest, ResultPointsAcc, [ + {error, {bad_timestamp, BadTs}} | ErrorPointsAcc + ]) + end; +lines_to_points(Data, [#{timestamp := Ts} = Item | Rest], ResultPointsAcc, ErrorPointsAcc) when + is_integer(Ts) +-> + continue_lines_to_points(Data, Item, Rest, ResultPointsAcc, ErrorPointsAcc). + +parse_timestamp([TsInt]) when is_integer(TsInt) -> + {ok, TsInt}; +parse_timestamp([TsBin]) -> + try + {ok, binary_to_integer(TsBin)} + catch + _:_ -> + {error, TsBin} + end. + +continue_lines_to_points(Data, Item, Rest, ResultPointsAcc, ErrorPointsAcc) -> + case line_to_point(Data, Item) of + #{fields := Fields} when map_size(Fields) =:= 0 -> + %% greptimedb client doesn't like empty field maps... + ErrorPointsAcc1 = [{error, no_fields} | ErrorPointsAcc], + lines_to_points(Data, Rest, ResultPointsAcc, ErrorPointsAcc1); + Point -> + lines_to_points(Data, Rest, [Point | ResultPointsAcc], ErrorPointsAcc) + end. + +line_to_point( + Data, + #{ + measurement := Measurement, + tags := Tags, + fields := Fields, + timestamp := Ts, + precision := Precision + } = Item +) -> + {_, EncodedTags} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Tags), + {_, EncodedFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields), + TableName = emqx_placeholder:proc_tmpl(Measurement, Data), + {TableName, [ + maps:without([precision], Item#{ + tags => EncodedTags, + fields => EncodedFields, + timestamp => maybe_convert_time_unit(Ts, Precision) + }) + ]}. + +maybe_convert_time_unit(Ts, {FromPrecision, ToPrecision}) -> + erlang:convert_time_unit(Ts, time_unit(FromPrecision), time_unit(ToPrecision)). + +time_unit(s) -> second; +time_unit(ms) -> millisecond; +time_unit(us) -> microsecond; +time_unit(ns) -> nanosecond. + +maps_config_to_data(K, V, {Data, Res}) -> + KTransOptions = #{return => rawlist, var_trans => fun key_filter/1}, + VTransOptions = #{return => rawlist, var_trans => fun data_filter/1}, + NK0 = emqx_placeholder:proc_tmpl(K, Data, KTransOptions), + NV = emqx_placeholder:proc_tmpl(V, Data, VTransOptions), + case {NK0, NV} of + {[undefined], _} -> + {Data, Res}; + %% undefined value in normal format [undefined] or int/uint format [undefined, <<"i">>] + {_, [undefined | _]} -> + {Data, Res}; + _ -> + NK = list_to_binary(NK0), + {Data, Res#{NK => value_type(NV)}} + end. + +value_type([Int, <<"i">>]) when + is_integer(Int) +-> + greptimedb_values:int64_value(Int); +value_type([UInt, <<"u">>]) when + is_integer(UInt) +-> + greptimedb_values:uint64_value(UInt); +value_type([Float]) when is_float(Float) -> + Float; +value_type([<<"t">>]) -> + greptimedb_values:boolean_value(true); +value_type([<<"T">>]) -> + greptimedb_values:boolean_value(true); +value_type([true]) -> + greptimedb_values:boolean_value(true); +value_type([<<"TRUE">>]) -> + greptimedb_values:boolean_value(true); +value_type([<<"True">>]) -> + greptimedb_values:boolean_value(true); +value_type([<<"f">>]) -> + greptimedb_values:boolean_value(false); +value_type([<<"F">>]) -> + greptimedb_values:boolean_value(false); +value_type([false]) -> + greptimedb_values:boolean_value(false); +value_type([<<"FALSE">>]) -> + greptimedb_values:boolean_value(false); +value_type([<<"False">>]) -> + greptimedb_values:boolean_value(false); +value_type(Val) -> + #{values => #{string_values => Val, datatype => 'STRING'}}. + +key_filter(undefined) -> undefined; +key_filter(Value) -> emqx_utils_conv:bin(Value). + +data_filter(undefined) -> undefined; +data_filter(Int) when is_integer(Int) -> Int; +data_filter(Number) when is_number(Number) -> Number; +data_filter(Bool) when is_boolean(Bool) -> Bool; +data_filter(Data) -> bin(Data). + +bin(Data) -> emqx_utils_conv:bin(Data). + +%% helper funcs +log_error_points(InstId, Errs) -> + lists:foreach( + fun({error, Reason}) -> + ?SLOG(error, #{ + msg => "greptimedb trans point failed", + connector => InstId, + reason => Reason + }) + end, + Errs + ). + +convert_server(<<"http://", Server/binary>>, HoconOpts) -> + convert_server(Server, HoconOpts); +convert_server(<<"https://", Server/binary>>, HoconOpts) -> + convert_server(Server, HoconOpts); +convert_server(Server, HoconOpts) -> + emqx_schema:convert_servers(Server, HoconOpts). + str(A) when is_atom(A) -> atom_to_list(A); str(B) when is_binary(B) -> binary_to_list(B); str(S) when is_list(S) -> S. + +is_unrecoverable_error({error, {unrecoverable_error, _}}) -> + true; +is_unrecoverable_error(_) -> + false. + +%%=================================================================== +%% eunit tests +%%=================================================================== + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). + +is_auth_key_test_() -> + [ + ?_assert(is_auth_key(<<"Authorization">>)), + ?_assertNot(is_auth_key(<<"Something">>)), + ?_assertNot(is_auth_key(89)) + ]. + +%% for coverage +desc_test_() -> + [ + ?_assertMatch( + {desc, _, _}, + desc(common) + ), + ?_assertMatch( + {desc, _, _}, + desc(greptimedb_grpc_v1) + ), + ?_assertMatch( + {desc, _, _}, + hocon_schema:field_schema(server(), desc) + ), + ?_assertMatch( + connector_greptimedb, + namespace() + ) + ]. +-endif. diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_tests.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_tests.erl new file mode 100644 index 000000000..a07ccd92d --- /dev/null +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_tests.erl @@ -0,0 +1,348 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_greptimedb_tests). + +-include_lib("eunit/include/eunit.hrl"). + +-define(INVALID_LINES, [ + " ", + " \n", + " \n\n\n ", + "\n", + " \n\n \n \n", + "measurement", + "measurement ", + "measurement,tag", + "measurement field", + "measurement,tag field", + "measurement,tag field ${timestamp}", + "measurement,tag=", + "measurement,tag=tag1", + "measurement,tag =", + "measurement field=", + "measurement field= ", + "measurement field = ", + "measurement, tag = field = ", + "measurement, tag = field = ", + "measurement, tag = tag_val field = field_val", + "measurement, tag = tag_val field = field_val ${timestamp}", + "measurement,= = ${timestamp}", + "measurement,t=a, f=a, ${timestamp}", + "measurement,t=a,t1=b, f=a,f1=b, ${timestamp}", + "measurement,t=a,t1=b, f=a,f1=b,", + "measurement,t=a, t1=b, f=a,f1=b,", + "measurement,t=a,,t1=b, f=a,f1=b,", + "measurement,t=a,,t1=b f=a,,f1=b", + "measurement,t=a,,t1=b f=a,f1=b ${timestamp}", + "measurement, f=a,f1=b", + "measurement, f=a,f1=b ${timestamp}", + "measurement,, f=a,f1=b ${timestamp}", + "measurement,, f=a,f1=b", + "measurement,, f=a,f1=b,, ${timestamp}", + "measurement f=a,f1=b,, ${timestamp}", + "measurement,t=a f=a,f1=b,, ${timestamp}", + "measurement,t=a f=a,f1=b,, ", + "measurement,t=a f=a,f1=b,,", + "measurement, t=a f=a,f1=b", + "measurement,t=a f=a, f1=b", + "measurement,t=a f=a, f1=b ${timestamp}", + "measurement, t=a f=a, f1=b ${timestamp}", + "measurement,t= a f=a,f1=b ${timestamp}", + "measurement,t= a f=a,f1 =b ${timestamp}", + "measurement, t = a f = a,f1 = b ${timestamp}", + "measurement,t=a f=a,f1=b \n ${timestamp}", + "measurement,t=a \n f=a,f1=b \n ${timestamp}", + "measurement,t=a \n f=a,f1=b \n ", + "\n measurement,t=a \n f=a,f1=b \n ${timestamp}", + "\n measurement,t=a \n f=a,f1=b \n", + %% not escaped backslash in a quoted field value is invalid + "measurement,tag=1 field=\"val\\1\"" +]). + +-define(VALID_LINE_PARSED_PAIRS, [ + {"m1,tag=tag1 field=field1 ${timestamp1}", #{ + measurement => "m1", + tags => [{"tag", "tag1"}], + fields => [{"field", "field1"}], + timestamp => "${timestamp1}" + }}, + {"m2,tag=tag2 field=field2", #{ + measurement => "m2", + tags => [{"tag", "tag2"}], + fields => [{"field", "field2"}], + timestamp => undefined + }}, + {"m3 field=field3 ${timestamp3}", #{ + measurement => "m3", + tags => [], + fields => [{"field", "field3"}], + timestamp => "${timestamp3}" + }}, + {"m4 field=field4", #{ + measurement => "m4", + tags => [], + fields => [{"field", "field4"}], + timestamp => undefined + }}, + {"m5,tag=tag5,tag_a=tag5a,tag_b=tag5b field=field5,field_a=field5a,field_b=field5b ${timestamp5}", + #{ + measurement => "m5", + tags => [{"tag", "tag5"}, {"tag_a", "tag5a"}, {"tag_b", "tag5b"}], + fields => [{"field", "field5"}, {"field_a", "field5a"}, {"field_b", "field5b"}], + timestamp => "${timestamp5}" + }}, + {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=field6,field_a=field6a,field_b=field6b", #{ + measurement => "m6", + tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], + fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + timestamp => undefined + }}, + {"m7,tag=tag7,tag_a=\"tag7a\",tag_b=tag7b field=\"field7\",field_a=field7a,field_b=\"field7b\"", + #{ + measurement => "m7", + tags => [{"tag", "tag7"}, {"tag_a", "\"tag7a\""}, {"tag_b", "tag7b"}], + fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b"}], + timestamp => undefined + }}, + {"m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a,field_b=\"field8b\" ${timestamp8}", + #{ + measurement => "m8", + tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], + fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "field8b"}], + timestamp => "${timestamp8}" + }}, + {"m9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}", + #{ + measurement => "m9", + tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}], + fields => [{"field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}], + timestamp => "${timestamp9}" + }}, + {"m10 field=\"\" ${timestamp10}", #{ + measurement => "m10", + tags => [], + fields => [{"field", ""}], + timestamp => "${timestamp10}" + }} +]). + +-define(VALID_LINE_EXTRA_SPACES_PARSED_PAIRS, [ + {"\n m1,tag=tag1 field=field1 ${timestamp1} \n", #{ + measurement => "m1", + tags => [{"tag", "tag1"}], + fields => [{"field", "field1"}], + timestamp => "${timestamp1}" + }}, + {" m2,tag=tag2 field=field2 ", #{ + measurement => "m2", + tags => [{"tag", "tag2"}], + fields => [{"field", "field2"}], + timestamp => undefined + }}, + {" m3 field=field3 ${timestamp3} ", #{ + measurement => "m3", + tags => [], + fields => [{"field", "field3"}], + timestamp => "${timestamp3}" + }}, + {" \n m4 field=field4\n ", #{ + measurement => "m4", + tags => [], + fields => [{"field", "field4"}], + timestamp => undefined + }}, + {" \n m5,tag=tag5,tag_a=tag5a,tag_b=tag5b field=field5,field_a=field5a,field_b=field5b ${timestamp5} \n", + #{ + measurement => "m5", + tags => [{"tag", "tag5"}, {"tag_a", "tag5a"}, {"tag_b", "tag5b"}], + fields => [{"field", "field5"}, {"field_a", "field5a"}, {"field_b", "field5b"}], + timestamp => "${timestamp5}" + }}, + {" m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=field6,field_a=field6a,field_b=field6b\n ", #{ + measurement => "m6", + tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], + fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + timestamp => undefined + }} +]). + +-define(VALID_LINE_PARSED_ESCAPED_CHARS_PAIRS, [ + {"m\\ =1\\,,\\,tag\\ \\==\\=tag\\ 1\\, \\,fie\\ ld\\ =\\ field\\,1 ${timestamp1}", #{ + measurement => "m =1,", + tags => [{",tag =", "=tag 1,"}], + fields => [{",fie ld ", " field,1"}], + timestamp => "${timestamp1}" + }}, + {"m2,tag=tag2 field=\"field \\\"2\\\",\n\"", #{ + measurement => "m2", + tags => [{"tag", "tag2"}], + fields => [{"field", "field \"2\",\n"}], + timestamp => undefined + }}, + {"m\\ 3 field=\"field3\" ${payload.timestamp\\ 3}", #{ + measurement => "m 3", + tags => [], + fields => [{"field", "field3"}], + timestamp => "${payload.timestamp 3}" + }}, + {"m4 field=\"\\\"field\\\\4\\\"\"", #{ + measurement => "m4", + tags => [], + fields => [{"field", "\"field\\4\""}], + timestamp => undefined + }}, + { + "m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5," + "field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5}", + #{ + measurement => "m5,mA", + tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}], + fields => [ + {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"} + ], + timestamp => "${timestamp5}" + } + }, + {"m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\"", + #{ + measurement => "m6", + tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], + fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + timestamp => undefined + }}, + { + "\\ \\ m7\\ \\ ,tag=\\ tag\\,7\\ ,tag_a=\"tag7a\",tag_b\\,tag1=tag7b field=\"field7\"," + "field_a=field7a,field_b=\"field7b\\\\\n\"", + #{ + measurement => " m7 ", + tags => [{"tag", " tag,7 "}, {"tag_a", "\"tag7a\""}, {"tag_b,tag1", "tag7b"}], + fields => [{"field", "field7"}, {"field_a", "field7a"}, {"field_b", "field7b\\\n"}], + timestamp => undefined + } + }, + { + "m8,tag=tag8,tag_a=\"tag8a\",tag_b=tag8b field=\"field8\",field_a=field8a," + "field_b=\"\\\"field\\\" = 8b\" ${timestamp8}", + #{ + measurement => "m8", + tags => [{"tag", "tag8"}, {"tag_a", "\"tag8a\""}, {"tag_b", "tag8b"}], + fields => [{"field", "field8"}, {"field_a", "field8a"}, {"field_b", "\"field\" = 8b"}], + timestamp => "${timestamp8}" + } + }, + {"m\\9,tag=tag9,tag_a=\"tag9a\",tag_b=tag9b field\\=field=\"field9\",field_a=field9a,field_b=\"\" ${timestamp9}", + #{ + measurement => "m\\9", + tags => [{"tag", "tag9"}, {"tag_a", "\"tag9a\""}, {"tag_b", "tag9b"}], + fields => [{"field=field", "field9"}, {"field_a", "field9a"}, {"field_b", ""}], + timestamp => "${timestamp9}" + }}, + {"m\\,10 \"field\\\\\"=\"\" ${timestamp10}", #{ + measurement => "m,10", + tags => [], + %% backslash should not be un-escaped in tag key + fields => [{"\"field\\\\\"", ""}], + timestamp => "${timestamp10}" + }} +]). + +-define(VALID_LINE_PARSED_ESCAPED_CHARS_EXTRA_SPACES_PAIRS, [ + {" \n m\\ =1\\,,\\,tag\\ \\==\\=tag\\ 1\\, \\,fie\\ ld\\ =\\ field\\,1 ${timestamp1} ", #{ + measurement => "m =1,", + tags => [{",tag =", "=tag 1,"}], + fields => [{",fie ld ", " field,1"}], + timestamp => "${timestamp1}" + }}, + {" m2,tag=tag2 field=\"field \\\"2\\\",\n\" ", #{ + measurement => "m2", + tags => [{"tag", "tag2"}], + fields => [{"field", "field \"2\",\n"}], + timestamp => undefined + }}, + {" m\\ 3 field=\"field3\" ${payload.timestamp\\ 3} ", #{ + measurement => "m 3", + tags => [], + fields => [{"field", "field3"}], + timestamp => "${payload.timestamp 3}" + }}, + {" m4 field=\"\\\"field\\\\4\\\"\" ", #{ + measurement => "m4", + tags => [], + fields => [{"field", "\"field\\4\""}], + timestamp => undefined + }}, + { + " m5\\,mA,tag=\\=tag5\\=,\\,tag_a\\,=tag\\ 5a,tag_b=tag5b \\ field\\ =field5," + "field\\ _\\ a=field5a,\\,field_b\\ =\\=\\,\\ field5b ${timestamp5} ", + #{ + measurement => "m5,mA", + tags => [{"tag", "=tag5="}, {",tag_a,", "tag 5a"}, {"tag_b", "tag5b"}], + fields => [ + {" field ", "field5"}, {"field _ a", "field5a"}, {",field_b ", "=, field5b"} + ], + timestamp => "${timestamp5}" + } + }, + {" m6,tag=tag6,tag_a=tag6a,tag_b=tag6b field=\"field6\",field_a=\"field6a\",field_b=\"field6b\" ", + #{ + measurement => "m6", + tags => [{"tag", "tag6"}, {"tag_a", "tag6a"}, {"tag_b", "tag6b"}], + fields => [{"field", "field6"}, {"field_a", "field6a"}, {"field_b", "field6b"}], + timestamp => undefined + }} +]). + +invalid_write_syntax_line_test_() -> + [?_assertThrow(_, to_influx_lines(L)) || L <- ?INVALID_LINES]. + +invalid_write_syntax_multiline_test_() -> + LinesList = [ + join("\n", ?INVALID_LINES), + join("\n\n\n", ?INVALID_LINES), + join("\n\n", lists:reverse(?INVALID_LINES)) + ], + [?_assertThrow(_, to_influx_lines(Lines)) || Lines <- LinesList]. + +valid_write_syntax_test_() -> + test_pairs(?VALID_LINE_PARSED_PAIRS). + +valid_write_syntax_with_extra_spaces_test_() -> + test_pairs(?VALID_LINE_EXTRA_SPACES_PARSED_PAIRS). + +valid_write_syntax_escaped_chars_test_() -> + test_pairs(?VALID_LINE_PARSED_ESCAPED_CHARS_PAIRS). + +valid_write_syntax_escaped_chars_with_extra_spaces_test_() -> + test_pairs(?VALID_LINE_PARSED_ESCAPED_CHARS_EXTRA_SPACES_PAIRS). + +test_pairs(PairsList) -> + {Lines, AllExpected} = lists:unzip(PairsList), + JoinedLines = join("\n", Lines), + JoinedLines1 = join("\n\n\n", Lines), + JoinedLines2 = join("\n\n", lists:reverse(Lines)), + SingleLineTests = + [ + ?_assertEqual([Expected], to_influx_lines(Line)) + || {Line, Expected} <- PairsList + ], + JoinedLinesTests = + [ + ?_assertEqual(AllExpected, to_influx_lines(JoinedLines)), + ?_assertEqual(AllExpected, to_influx_lines(JoinedLines1)), + ?_assertEqual(lists:reverse(AllExpected), to_influx_lines(JoinedLines2)) + ], + SingleLineTests ++ JoinedLinesTests. + +join(Sep, LinesList) -> + lists:flatten(lists:join(Sep, LinesList)). + +to_influx_lines(RawLines) -> + OldLevel = emqx_logger:get_primary_log_level(), + try + %% mute error logs from this call + emqx_logger:set_primary_log_level(none), + emqx_bridge_greptimedb:to_influx_lines(RawLines) + after + emqx_logger:set_primary_log_level(OldLevel) + end. From 89bce998704a0fedb8f434cee809dd1abf777c76 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Thu, 6 Jul 2023 19:11:20 +0800 Subject: [PATCH 06/21] test: greptimedb data brige --- .tool-versions | 2 +- apps/emqx_bridge/src/emqx_bridge.erl | 2 +- .../src/schema/emqx_bridge_enterprise.erl | 23 +- apps/emqx_bridge_greptimedb/docker-ct | 2 + apps/emqx_bridge_greptimedb/rebar.config | 2 +- .../src/emqx_bridge_greptimedb.erl | 4 +- .../src/emqx_bridge_greptimedb_connector.erl | 20 +- .../test/emqx_bridge_greptimedb_SUITE.erl | 1003 +++++++++++++++++ 8 files changed, 1043 insertions(+), 15 deletions(-) create mode 100644 apps/emqx_bridge_greptimedb/docker-ct create mode 100644 apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl diff --git a/.tool-versions b/.tool-versions index 3a2251dc8..0dbab2a1d 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 25.3.2-1 +erlang 25.3.2.3 elixir 1.14.5-otp-25 diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index 612481663..b60276910 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -90,7 +90,7 @@ T == oracle; T == iotdb; T == kinesis_producer; - T == greptimedb + T == greptimedb_grpc_v1 ). -define(ROOT_KEY, bridges). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index 7b9e1d4fa..048dcbf90 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -50,7 +50,7 @@ api_schemas(Method) -> api_ref(emqx_bridge_iotdb, <<"iotdb">>, Method), api_ref(emqx_bridge_rabbitmq, <<"rabbitmq">>, Method), api_ref(emqx_bridge_kinesis, <<"kinesis_producer">>, Method ++ "_producer"), - api_ref(emqx_bridge_greptimedb, Method) + api_ref(emqx_bridge_greptimedb, <<"greptimedb_grpc_v1">>, Method ++ "_grpc_v1") ]. schema_modules() -> @@ -124,8 +124,7 @@ resource_type(oracle) -> emqx_oracle; resource_type(iotdb) -> emqx_bridge_iotdb_impl; resource_type(rabbitmq) -> emqx_bridge_rabbitmq_connector; resource_type(kinesis_producer) -> emqx_bridge_kinesis_impl_producer. -resource_type(rabbitmq) -> emqx_bridge_rabbitmq_connector. -resource_type(greptimedb) -> emqx_bridge_greptimedb_connector. +resource_type(greptimedb_grpc_v1) -> emqx_bridge_greptimedb_connector. fields(bridges) -> [ @@ -214,7 +213,8 @@ fields(bridges) -> influxdb_structs() ++ redis_structs() ++ pgsql_structs() ++ clickhouse_structs() ++ sqlserver_structs() ++ rabbitmq_structs() ++ - kinesis_structs(). + kinesis_structs() ++ + greptimedb_structs(). mongodb_structs() -> [ @@ -299,6 +299,21 @@ influxdb_structs() -> ] ]. +greptimedb_structs() -> + [ + {Protocol, + mk( + hoconsc:map(name, ref(emqx_bridge_greptimedb, Protocol)), + #{ + desc => <<"GreptimeDB Bridge Config">>, + required => false + } + )} + || Protocol <- [ + greptimedb_grpc_v1 + ] + ]. + redis_structs() -> [ {Type, diff --git a/apps/emqx_bridge_greptimedb/docker-ct b/apps/emqx_bridge_greptimedb/docker-ct new file mode 100644 index 000000000..1a9647132 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/docker-ct @@ -0,0 +1,2 @@ +toxiproxy +greptimedb diff --git a/apps/emqx_bridge_greptimedb/rebar.config b/apps/emqx_bridge_greptimedb/rebar.config index 952281286..2001a72fc 100644 --- a/apps/emqx_bridge_greptimedb/rebar.config +++ b/apps/emqx_bridge_greptimedb/rebar.config @@ -6,7 +6,7 @@ {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}}, - {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.1"}}} + {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {branch, "feature/check-auth"}}} ]}. {plugins, [rebar3_path_deps]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl index f37ddf320..5bd8f6852 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -47,7 +47,7 @@ values("greptimedb_grpc_v1", post) -> bucket => <<"example_bucket">>, org => <<"examlpe_org">>, token => <<"example_token">>, - server => <<"127.0.0.1:4000">> + server => <<"127.0.0.1:4001">> }, values(common, "greptimedb_grpc_v1", SupportUint, TypeOpts); values(Protocol, put) -> @@ -68,7 +68,7 @@ values(common, Protocol, SupportUint, TypeOpts) -> batch_size => 100, batch_time => <<"20ms">> }, - server => <<"127.0.0.1:4000">>, + server => <<"127.0.0.1:4001">>, ssl => #{enable => false} }, maps:merge(TypeOpts, CommonConfigs). diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index a02df09c5..bc4eacbab 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -39,7 +39,7 @@ %% Allocatable resources -define(greptime_client, greptime_client). --define(GREPTIMEDB_DEFAULT_PORT, 4000). +-define(GREPTIMEDB_DEFAULT_PORT, 4001). -define(DEFAULT_DB, <<"public">>). @@ -81,7 +81,7 @@ on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, c #{batch => false, mode => sync, error => ErrorPoints} ), log_error_points(InstId, ErrorPoints), - {error, {unrecoverable_error, ErrorPoints}} + ErrorPoints end. %% Once a Batched Data trans to points failed. @@ -140,13 +140,21 @@ fields(common) -> fields(greptimedb_grpc_v1) -> fields(common) ++ [ - {dbname, mk(binary(), #{required => true, desc => ?DESC("dbname")})} + {dbname, mk(binary(), #{required => true, desc => ?DESC("dbname")})}, + {username, mk(binary(), #{desc => ?DESC("username")})}, + {password, + mk(binary(), #{ + desc => ?DESC("password"), + format => <<"password">>, + sensitive => true, + converter => fun emqx_schema:password_converter/2 + })} ] ++ emqx_connector_schema_lib:ssl_fields(). server() -> Meta = #{ required => false, - default => <<"127.0.0.1:4000">>, + default => <<"127.0.0.1:4001">>, desc => ?DESC("server"), converter => fun convert_server/2 }, @@ -477,7 +485,7 @@ line_to_point( {_, EncodedFields} = maps:fold(fun maps_config_to_data/3, {Data, #{}}, Fields), TableName = emqx_placeholder:proc_tmpl(Measurement, Data), {TableName, [ - maps:without([precision], Item#{ + maps:without([precision, measurement], Item#{ tags => EncodedTags, fields => EncodedFields, timestamp => maybe_convert_time_unit(Ts, Precision) @@ -539,7 +547,7 @@ value_type([<<"FALSE">>]) -> value_type([<<"False">>]) -> greptimedb_values:boolean_value(false); value_type(Val) -> - #{values => #{string_values => Val, datatype => 'STRING'}}. + #{values => #{string_values => Val}, datatype => 'STRING'}. key_filter(undefined) -> undefined; key_filter(Value) -> emqx_utils_conv:bin(Value). diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl new file mode 100644 index 000000000..e694060f5 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -0,0 +1,1003 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- +-module(emqx_bridge_greptimedb_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). +-include_lib("snabbkaffe/include/snabbkaffe.hrl"). + +%%------------------------------------------------------------------------------ +%% CT boilerplate +%%------------------------------------------------------------------------------ + +all() -> + [ + {group, with_batch}, + {group, without_batch} + ]. + +groups() -> + TCs = emqx_common_test_helpers:all(?MODULE), + [ + {with_batch, [ + {group, sync_query} + ]}, + {without_batch, [ + {group, sync_query} + ]}, + {sync_query, [ + {group, grpcv1_tcp}, + {group, grpcv1_tls} + ]}, + {grpcv1_tcp, TCs}, + {grpcv1_tls, TCs} + ]. + +init_per_suite(Config) -> + Config. + +end_per_suite(_Config) -> + delete_all_bridges(), + emqx_mgmt_api_test_util:end_suite(), + ok = emqx_connector_test_helpers:stop_apps([ + emqx_conf, emqx_bridge, emqx_resource, emqx_rule_engine + ]), + _ = application:stop(emqx_connector), + ok. + +init_per_group(GreptimedbType, Config0) when + GreptimedbType =:= grpcv1_tcp; + GreptimedbType =:= grpcv1_tls +-> + #{ + host := GreptimedbHost, + port := GreptimedbPort, + use_tls := UseTLS, + proxy_name := ProxyName + } = + case GreptimedbType of + grpcv1_tcp -> + #{ + host => os:getenv("GREPTIMEDB_GRPCV1_TCP_HOST", "toxiproxy"), + port => list_to_integer(os:getenv("GREPTIMEDB_GRPCV1_TCP_PORT", "4001")), + use_tls => false, + proxy_name => "greptimedb_tcp" + }; + grpcv1_tls -> + #{ + host => os:getenv("GREPTIMEDB_GRPCV1_TLS_HOST", "toxiproxy"), + port => list_to_integer(os:getenv("GREPTIMEDB_GRPCV1_TLS_PORT", "4001")), + use_tls => true, + proxy_name => "greptimedb_tls" + } + end, + case emqx_common_test_helpers:is_tcp_server_available(GreptimedbHost, GreptimedbPort) of + true -> + ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), + ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ok = start_apps(), + {ok, _} = application:ensure_all_started(emqx_connector), + application:ensure_all_started(greptimedb), + emqx_mgmt_api_test_util:init_suite(), + Config = [{use_tls, UseTLS} | Config0], + {Name, ConfigString, GreptimedbConfig} = greptimedb_config( + grpcv1, GreptimedbHost, GreptimedbPort, Config + ), + EHttpcPoolNameBin = <<(atom_to_binary(?MODULE))/binary, "_grpcv1">>, + EHttpcPoolName = binary_to_atom(EHttpcPoolNameBin), + {EHttpcTransport, EHttpcTransportOpts} = + case UseTLS of + true -> {tls, [{verify, verify_none}]}; + false -> {tcp, []} + end, + EHttpcPoolOpts = [ + {host, GreptimedbHost}, + {port, GreptimedbPort}, + {pool_size, 1}, + {transport, EHttpcTransport}, + {transport_opts, EHttpcTransportOpts} + ], + {ok, _} = ehttpc_sup:start_pool(EHttpcPoolName, EHttpcPoolOpts), + [ + {proxy_host, ProxyHost}, + {proxy_port, ProxyPort}, + {proxy_name, ProxyName}, + {greptimedb_host, GreptimedbHost}, + {greptimedb_port, GreptimedbPort}, + {greptimedb_type, grpcv1}, + {greptimedb_config, GreptimedbConfig}, + {greptimedb_config_string, ConfigString}, + {ehttpc_pool_name, EHttpcPoolName}, + {greptimedb_name, Name} + | Config + ]; + false -> + {skip, no_greptimedb} + end; +init_per_group(sync_query, Config) -> + [{query_mode, sync} | Config]; +init_per_group(with_batch, Config) -> + [{batch_size, 100} | Config]; +init_per_group(without_batch, Config) -> + [{batch_size, 1} | Config]; +init_per_group(_Group, Config) -> + Config. + +end_per_group(Group, Config) when + Group =:= grpcv1_tcp; + Group =:= grpcv1_tls +-> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + EHttpcPoolName = ?config(ehttpc_pool_name, Config), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + ehttpc_sup:stop_pool(EHttpcPoolName), + delete_bridge(Config), + ok; +end_per_group(_Group, _Config) -> + ok. + +init_per_testcase(_Testcase, Config) -> + delete_all_rules(), + delete_all_bridges(), + Config. + +end_per_testcase(_Testcase, Config) -> + ProxyHost = ?config(proxy_host, Config), + ProxyPort = ?config(proxy_port, Config), + ok = snabbkaffe:stop(), + emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), + delete_all_rules(), + delete_all_bridges(), + ok. + +%%------------------------------------------------------------------------------ +%% Helper fns +%%------------------------------------------------------------------------------ + +start_apps() -> + %% some configs in emqx_conf app are mandatory + %% we want to make sure they are loaded before + %% ekka start in emqx_common_test_helpers:start_apps/1 + emqx_common_test_helpers:render_and_load_app_config(emqx_conf), + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource, emqx_bridge, emqx_rule_engine]). + +example_write_syntax() -> + %% N.B.: this single space character is relevant + <<"${topic},clientid=${clientid}", " ", "payload=${payload},", + "${clientid}_int_value=${payload.int_key}i,", + "uint_value=${payload.uint_key}u," + "float_value=${payload.float_key},", "undef_value=${payload.undef},", + "${undef_key}=\"hard-coded-value\",", "bool=${payload.bool}">>. + +greptimedb_config(grpcv1 = Type, GreptimedbHost, GreptimedbPort, Config) -> + BatchSize = proplists:get_value(batch_size, Config, 100), + QueryMode = proplists:get_value(query_mode, Config, sync), + UseTLS = proplists:get_value(use_tls, Config, false), + Name = atom_to_binary(?MODULE), + WriteSyntax = example_write_syntax(), + ConfigString = + io_lib:format( + "bridges.greptimedb_grpc_v1.~s {\n" + " enable = true\n" + " server = \"~p:~b\"\n" + " dbname = public\n" + " username = greptime_user\n" + " password = greptime_pwd\n" + " precision = ns\n" + " write_syntax = \"~s\"\n" + " resource_opts = {\n" + " request_ttl = 1s\n" + " query_mode = ~s\n" + " batch_size = ~b\n" + " }\n" + " ssl {\n" + " enable = ~p\n" + " verify = verify_none\n" + " }\n" + "}\n", + [ + Name, + GreptimedbHost, + GreptimedbPort, + WriteSyntax, + QueryMode, + BatchSize, + UseTLS + ] + ), + {Name, ConfigString, parse_and_check(ConfigString, Type, Name)}. + +parse_and_check(ConfigString, Type, Name) -> + {ok, RawConf} = hocon:binary(ConfigString, #{format => map}), + TypeBin = greptimedb_type_bin(Type), + hocon_tconf:check_plain(emqx_bridge_schema, RawConf, #{required => false, atom_key => false}), + #{<<"bridges">> := #{TypeBin := #{Name := Config}}} = RawConf, + Config. + +greptimedb_type_bin(grpcv1) -> + <<"greptimedb_grpc_v1">>. + +create_bridge(Config) -> + create_bridge(Config, _Overrides = #{}). + +create_bridge(Config, Overrides) -> + Type = greptimedb_type_bin(?config(greptimedb_type, Config)), + Name = ?config(greptimedb_name, Config), + GreptimedbConfig0 = ?config(greptimedb_config, Config), + GreptimedbConfig = emqx_utils_maps:deep_merge(GreptimedbConfig0, Overrides), + emqx_bridge:create(Type, Name, GreptimedbConfig). + +delete_bridge(Config) -> + Type = greptimedb_type_bin(?config(greptimedb_type, Config)), + Name = ?config(greptimedb_name, Config), + emqx_bridge:remove(Type, Name). + +delete_all_bridges() -> + lists:foreach( + fun(#{name := Name, type := Type}) -> + emqx_bridge:remove(Type, Name) + end, + emqx_bridge:list() + ). + +delete_all_rules() -> + lists:foreach( + fun(#{id := RuleId}) -> + ok = emqx_rule_engine:delete_rule(RuleId) + end, + emqx_rule_engine:get_rules() + ). + +create_rule_and_action_http(Config) -> + create_rule_and_action_http(Config, _Overrides = #{}). + +create_rule_and_action_http(Config, Overrides) -> + GreptimedbName = ?config(greptimedb_name, Config), + Type = greptimedb_type_bin(?config(greptimedb_type, Config)), + BridgeId = emqx_bridge_resource:bridge_id(Type, GreptimedbName), + Params0 = #{ + enable => true, + sql => <<"SELECT * FROM \"t/topic\"">>, + actions => [BridgeId] + }, + Params = emqx_utils_maps:deep_merge(Params0, Overrides), + Path = emqx_mgmt_api_test_util:api_path(["rules"]), + AuthHeader = emqx_mgmt_api_test_util:auth_header_(), + case emqx_mgmt_api_test_util:request_api(post, Path, "", AuthHeader, Params) of + {ok, Res} -> {ok, emqx_utils_json:decode(Res, [return_maps])}; + Error -> Error + end. + +send_message(Config, Payload) -> + Name = ?config(greptimedb_name, Config), + Type = greptimedb_type_bin(?config(greptimedb_type, Config)), + BridgeId = emqx_bridge_resource:bridge_id(Type, Name), + emqx_bridge:send_message(BridgeId, Payload). + +query_by_clientid(ClientId, Config) -> + GreptimedbHost = ?config(greptimedb_host, Config), + GreptimedbPort = ?config(greptimedb_port, Config), + EHttpcPoolName = ?config(ehttpc_pool_name, Config), + UseTLS = ?config(use_tls, Config), + Path = <<"/api/v2/query?org=emqx">>, + Scheme = + case UseTLS of + true -> <<"https://">>; + false -> <<"http://">> + end, + URI = iolist_to_binary([ + Scheme, + list_to_binary(GreptimedbHost), + ":", + integer_to_binary(GreptimedbPort), + Path + ]), + Query = + << + "from(bucket: \"mqtt\")\n" + " |> range(start: -12h)\n" + " |> filter(fn: (r) => r.clientid == \"", + ClientId/binary, + "\")" + >>, + Headers = [ + {"Authorization", "Token abcdefg"}, + {"Content-Type", "application/json"} + ], + Body = + emqx_utils_json:encode(#{ + query => Query, + dialect => #{ + header => true, + delimiter => <<";">> + } + }), + {ok, 200, _Headers, RawBody0} = + ehttpc:request( + EHttpcPoolName, + post, + {URI, Headers, Body}, + _Timeout = 10_000, + _Retry = 0 + ), + RawBody1 = iolist_to_binary(string:replace(RawBody0, <<"\r\n">>, <<"\n">>, all)), + {ok, DecodedCSV0} = erl_csv:decode(RawBody1, #{separator => <<$;>>}), + DecodedCSV1 = [ + [Field || Field <- Line, Field =/= <<>>] + || Line <- DecodedCSV0, + Line =/= [<<>>] + ], + DecodedCSV2 = csv_lines_to_maps(DecodedCSV1, []), + index_by_field(DecodedCSV2). + +decode_csv(RawBody) -> + Lines = + [ + binary:split(Line, [<<";">>], [global, trim_all]) + || Line <- binary:split(RawBody, [<<"\r\n">>], [global, trim_all]) + ], + csv_lines_to_maps(Lines, []). + +csv_lines_to_maps([Fields, Data | Rest], Acc) -> + Map = maps:from_list(lists:zip(Fields, Data)), + csv_lines_to_maps(Rest, [Map | Acc]); +csv_lines_to_maps(_Data, Acc) -> + lists:reverse(Acc). + +index_by_field(DecodedCSV) -> + maps:from_list([{Field, Data} || Data = #{<<"_field">> := Field} <- DecodedCSV]). + +assert_persisted_data(ClientId, Expected, PersistedData) -> + ClientIdIntKey = <>, + maps:foreach( + fun + (int_value, ExpectedValue) -> + ?assertMatch( + #{<<"_value">> := ExpectedValue}, + maps:get(ClientIdIntKey, PersistedData) + ); + (Key, ExpectedValue) -> + ?assertMatch( + #{<<"_value">> := ExpectedValue}, + maps:get(atom_to_binary(Key), PersistedData), + #{expected => ExpectedValue} + ) + end, + Expected + ), + ok. + +resource_id(Config) -> + Type = greptimedb_type_bin(?config(greptimedb_type, Config)), + Name = ?config(greptimedb_name, Config), + emqx_bridge_resource:resource_id(Type, Name). + +%%------------------------------------------------------------------------------ +%% Testcases +%%------------------------------------------------------------------------------ + +t_start_ok(Config) -> + QueryMode = ?config(query_mode, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = #{ + int_key => -123, + bool => true, + float_key => 24.5, + uint_key => 123 + }, + SentData = #{ + <<"clientid">> => ClientId, + <<"topic">> => atom_to_binary(?FUNCTION_NAME), + <<"payload">> => Payload, + <<"timestamp">> => erlang:system_time(millisecond) + }, + ?check_trace( + begin + case QueryMode of + sync -> + ?assertMatch(ok, send_message(Config, SentData)) + end, + PersistedData = query_by_clientid(ClientId, Config), + Expected = #{ + bool => <<"true">>, + int_value => <<"-123">>, + uint_value => <<"123">>, + float_value => <<"24.5">>, + payload => emqx_utils_json:encode(Payload) + }, + assert_persisted_data(ClientId, Expected, PersistedData), + ok + end, + fun(Trace0) -> + Trace = ?of_kind(greptimedb_connector_send_query, Trace0), + ?assertMatch([#{points := [_]}], Trace), + [#{points := [Point]}] = Trace, + ct:pal("sent point: ~p", [Point]), + ?assertMatch( + #{ + fields := #{}, + measurement := <<_/binary>>, + tags := #{}, + timestamp := TS + } when is_integer(TS), + Point + ), + #{fields := Fields} = Point, + ?assert(lists:all(fun is_binary/1, maps:keys(Fields))), + ?assertNot(maps:is_key(<<"undefined">>, Fields)), + ?assertNot(maps:is_key(<<"undef_value">>, Fields)), + ok + end + ), + ok. + +t_start_already_started(Config) -> + Type = greptimedb_type_bin(?config(greptimedb_type, Config)), + Name = ?config(greptimedb_name, Config), + GreptimedbConfigString = ?config(greptimedb_config_string, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + ResourceId = resource_id(Config), + TypeAtom = binary_to_atom(Type), + NameAtom = binary_to_atom(Name), + {ok, #{bridges := #{TypeAtom := #{NameAtom := GreptimedbConfigMap}}}} = emqx_hocon:check( + emqx_bridge_schema, GreptimedbConfigString + ), + ?check_trace( + emqx_bridge_greptimedb_connector:on_start(ResourceId, GreptimedbConfigMap), + fun(Result, Trace) -> + ?assertMatch({ok, _}, Result), + ?assertMatch([_], ?of_kind(greptimedb_connector_start_already_started, Trace)), + ok + end + ), + ok. + +t_start_ok_timestamp_write_syntax(Config) -> + GreptimedbType = ?config(greptimedb_type, Config), + GreptimedbName = ?config(greptimedb_name, Config), + GreptimedbConfigString0 = ?config(greptimedb_config_string, Config), + GreptimedbTypeCfg = + case GreptimedbType of + grpcv1 -> "greptimedb_grpc_v1" + end, + WriteSyntax = + %% N.B.: this single space characters are relevant + <<"${topic},clientid=${clientid}", " ", "payload=${payload},", + "${clientid}_int_value=${payload.int_key}i,", + "uint_value=${payload.uint_key}u," + "bool=${payload.bool}", " ", "${timestamp}">>, + %% append this to override the config + GreptimedbConfigString1 = + io_lib:format( + "bridges.~s.~s {\n" + " write_syntax = \"~s\"\n" + "}\n", + [GreptimedbTypeCfg, GreptimedbName, WriteSyntax] + ), + GreptimedbConfig1 = parse_and_check( + GreptimedbConfigString0 ++ GreptimedbConfigString1, + GreptimedbType, + GreptimedbName + ), + Config1 = [{greptimedb_config, GreptimedbConfig1} | Config], + ?assertMatch( + {ok, _}, + create_bridge(Config1) + ), + ok. + +t_start_ok_no_subject_tags_write_syntax(Config) -> + GreptimedbType = ?config(greptimedb_type, Config), + GreptimedbName = ?config(greptimedb_name, Config), + GreptimedbConfigString0 = ?config(greptimedb_config_string, Config), + GreptimedbTypeCfg = + case GreptimedbType of + grpcv1 -> "greptimedb_grpc_v1" + end, + WriteSyntax = + %% N.B.: this single space characters are relevant + <<"${topic}", " ", "payload=${payload},", "${clientid}_int_value=${payload.int_key}i,", + "uint_value=${payload.uint_key}u," + "bool=${payload.bool}", " ", "${timestamp}">>, + %% append this to override the config + GreptimedbConfigString1 = + io_lib:format( + "bridges.~s.~s {\n" + " write_syntax = \"~s\"\n" + "}\n", + [GreptimedbTypeCfg, GreptimedbName, WriteSyntax] + ), + GreptimedbConfig1 = parse_and_check( + GreptimedbConfigString0 ++ GreptimedbConfigString1, + GreptimedbType, + GreptimedbName + ), + Config1 = [{greptimedb_config, GreptimedbConfig1} | Config], + ?assertMatch( + {ok, _}, + create_bridge(Config1) + ), + ok. + +t_const_timestamp(Config) -> + QueryMode = ?config(query_mode, Config), + Const = erlang:system_time(nanosecond), + ConstBin = integer_to_binary(Const), + TsStr = iolist_to_binary( + calendar:system_time_to_rfc3339(Const, [{unit, nanosecond}, {offset, "Z"}]) + ), + ?assertMatch( + {ok, _}, + create_bridge( + Config, + #{ + <<"write_syntax">> => + <<"mqtt,clientid=${clientid} foo=${payload.foo}i,bar=5i ", ConstBin/binary>> + } + ) + ), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = #{<<"foo">> => 123}, + SentData = #{ + <<"clientid">> => ClientId, + <<"topic">> => atom_to_binary(?FUNCTION_NAME), + <<"payload">> => Payload, + <<"timestamp">> => erlang:system_time(millisecond) + }, + case QueryMode of + sync -> + ?assertMatch(ok, send_message(Config, SentData)) + end, + PersistedData = query_by_clientid(ClientId, Config), + Expected = #{foo => <<"123">>}, + assert_persisted_data(ClientId, Expected, PersistedData), + TimeReturned0 = maps:get(<<"_time">>, maps:get(<<"foo">>, PersistedData)), + TimeReturned = pad_zero(TimeReturned0), + ?assertEqual(TsStr, TimeReturned). + +%% greptimedb returns timestamps without trailing zeros such as +%% "2023-02-28T17:21:51.63678163Z" +%% while the standard should be +%% "2023-02-28T17:21:51.636781630Z" +pad_zero(BinTs) -> + StrTs = binary_to_list(BinTs), + [Nano | Rest] = lists:reverse(string:tokens(StrTs, ".")), + [$Z | NanoNum] = lists:reverse(Nano), + Padding = lists:duplicate(10 - length(Nano), $0), + NewNano = lists:reverse(NanoNum) ++ Padding ++ "Z", + iolist_to_binary(string:join(lists:reverse([NewNano | Rest]), ".")). + +t_boolean_variants(Config) -> + QueryMode = ?config(query_mode, Config), + ?assertMatch( + {ok, _}, + create_bridge(Config) + ), + BoolVariants = #{ + true => true, + false => false, + <<"t">> => true, + <<"f">> => false, + <<"T">> => true, + <<"F">> => false, + <<"TRUE">> => true, + <<"FALSE">> => false, + <<"True">> => true, + <<"False">> => false + }, + maps:foreach( + fun(BoolVariant, Translation) -> + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = #{ + int_key => -123, + bool => BoolVariant, + uint_key => 123 + }, + SentData = #{ + <<"clientid">> => ClientId, + <<"topic">> => atom_to_binary(?FUNCTION_NAME), + <<"timestamp">> => erlang:system_time(millisecond), + <<"payload">> => Payload + }, + case QueryMode of + sync -> + ?assertMatch(ok, send_message(Config, SentData)) + end, + case QueryMode of + sync -> ok + end, + PersistedData = query_by_clientid(ClientId, Config), + Expected = #{ + bool => atom_to_binary(Translation), + int_value => <<"-123">>, + uint_value => <<"123">>, + payload => emqx_utils_json:encode(Payload) + }, + assert_persisted_data(ClientId, Expected, PersistedData), + ok + end, + BoolVariants + ), + ok. + +t_bad_timestamp(Config) -> + GreptimedbType = ?config(greptimedb_type, Config), + GreptimedbName = ?config(greptimedb_name, Config), + QueryMode = ?config(query_mode, Config), + BatchSize = ?config(batch_size, Config), + GreptimedbConfigString0 = ?config(greptimedb_config_string, Config), + GreptimedbTypeCfg = + case GreptimedbType of + grpcv1 -> "greptimedb_grpc_v1" + end, + WriteSyntax = + %% N.B.: this single space characters are relevant + <<"${topic}", " ", "payload=${payload},", "${clientid}_int_value=${payload.int_key}i,", + "uint_value=${payload.uint_key}u," + "bool=${payload.bool}", " ", "bad_timestamp">>, + %% append this to override the config + GreptimedbConfigString1 = + io_lib:format( + "bridges.~s.~s {\n" + " write_syntax = \"~s\"\n" + "}\n", + [GreptimedbTypeCfg, GreptimedbName, WriteSyntax] + ), + GreptimedbConfig1 = parse_and_check( + GreptimedbConfigString0 ++ GreptimedbConfigString1, + GreptimedbType, + GreptimedbName + ), + Config1 = [{greptimedb_config, GreptimedbConfig1} | Config], + ?assertMatch( + {ok, _}, + create_bridge(Config1) + ), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = #{ + int_key => -123, + bool => false, + uint_key => 123 + }, + SentData = #{ + <<"clientid">> => ClientId, + <<"topic">> => atom_to_binary(?FUNCTION_NAME), + <<"timestamp">> => erlang:system_time(millisecond), + <<"payload">> => Payload + }, + ?check_trace( + ?wait_async_action( + send_message(Config1, SentData), + #{?snk_kind := greptimedb_connector_send_query_error}, + 10_000 + ), + fun(Result, _Trace) -> + ?assertMatch({_, {ok, _}}, Result), + {Return, {ok, _}} = Result, + IsBatch = BatchSize > 1, + case {QueryMode, IsBatch} of + {sync, false} -> + ?assertEqual( + {error, [ + {error, {bad_timestamp, <<"bad_timestamp">>}} + ]}, + Return + ); + {sync, true} -> + ?assertEqual({error, {unrecoverable_error, points_trans_failed}}, Return) + end, + ok + end + ), + ok. + +t_get_status(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + {ok, _} = create_bridge(Config), + ResourceId = resource_id(Config), + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)), + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceId)) + end), + ok. + +t_create_disconnected(Config) -> + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + ProxyName = ?config(proxy_name, Config), + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + ?assertMatch({ok, _}, create_bridge(Config)) + end), + fun(Trace) -> + ?assertMatch( + [#{error := greptimedb_client_not_alive, reason := econnrefused}], + ?of_kind(greptimedb_connector_start_failed, Trace) + ), + ok + end + ), + ok. + +t_start_error(Config) -> + %% simulate client start error + ?check_trace( + emqx_common_test_helpers:with_mock( + greptimedb, + start_client, + fun(_Config) -> {error, some_error} end, + fun() -> + ?wait_async_action( + ?assertMatch({ok, _}, create_bridge(Config)), + #{?snk_kind := greptimedb_connector_start_failed}, + 10_000 + ) + end + ), + fun(Trace) -> + ?assertMatch( + [#{error := some_error}], + ?of_kind(greptimedb_connector_start_failed, Trace) + ), + ok + end + ), + ok. + +t_start_exception(Config) -> + %% simulate client start exception + ?check_trace( + emqx_common_test_helpers:with_mock( + greptimedb, + start_client, + fun(_Config) -> error(boom) end, + fun() -> + ?wait_async_action( + ?assertMatch({ok, _}, create_bridge(Config)), + #{?snk_kind := greptimedb_connector_start_exception}, + 10_000 + ) + end + ), + fun(Trace) -> + ?assertMatch( + [#{error := {error, boom}}], + ?of_kind(greptimedb_connector_start_exception, Trace) + ), + ok + end + ), + ok. + +t_write_failure(Config) -> + ProxyName = ?config(proxy_name, Config), + ProxyPort = ?config(proxy_port, Config), + ProxyHost = ?config(proxy_host, Config), + QueryMode = ?config(query_mode, Config), + {ok, _} = create_bridge(Config), + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = #{ + int_key => -123, + bool => true, + float_key => 24.5, + uint_key => 123 + }, + SentData = #{ + <<"clientid">> => ClientId, + <<"topic">> => atom_to_binary(?FUNCTION_NAME), + <<"timestamp">> => erlang:system_time(millisecond), + <<"payload">> => Payload + }, + ?check_trace( + emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> + case QueryMode of + sync -> + {_, {ok, _}} = + ?wait_async_action( + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + send_message(Config, SentData) + ), + #{?snk_kind := handle_async_reply, action := nack}, + 1_000 + ) + end + end), + fun(Trace0) -> + case QueryMode of + sync -> + Trace = ?of_kind(handle_async_reply, Trace0), + ?assertMatch([_ | _], Trace), + [#{result := Result} | _] = Trace, + ?assert( + not emqx_bridge_greptimedb_connector:is_unrecoverable_error(Result), + #{got => Result} + ) + end, + ok + end + ), + ok. + +t_missing_field(Config) -> + BatchSize = ?config(batch_size, Config), + IsBatch = BatchSize > 1, + {ok, _} = + create_bridge( + Config, + #{ + <<"resource_opts">> => #{<<"worker_pool_size">> => 1}, + <<"write_syntax">> => <<"${clientid} foo=${foo}i">> + } + ), + %% note: we don't select foo here, but we interpolate it in the + %% fields, so it'll become undefined. + {ok, _} = create_rule_and_action_http(Config, #{sql => <<"select * from \"t/topic\"">>}), + ClientId0 = emqx_guid:to_hexstr(emqx_guid:gen()), + ClientId1 = emqx_guid:to_hexstr(emqx_guid:gen()), + %% Message with the field that we "forgot" to select in the rule + Msg0 = emqx_message:make(ClientId0, <<"t/topic">>, emqx_utils_json:encode(#{foo => 123})), + %% Message without any fields + Msg1 = emqx_message:make(ClientId1, <<"t/topic">>, emqx_utils_json:encode(#{})), + ?check_trace( + begin + emqx:publish(Msg0), + emqx:publish(Msg1), + NEvents = 1, + {ok, _} = + snabbkaffe:block_until( + ?match_n_events(NEvents, #{ + ?snk_kind := greptimedb_connector_send_query_error + }), + _Timeout1 = 10_000 + ), + ok + end, + fun(Trace) -> + PersistedData0 = query_by_clientid(ClientId0, Config), + PersistedData1 = query_by_clientid(ClientId1, Config), + case IsBatch of + true -> + ?assertMatch( + [#{error := points_trans_failed} | _], + ?of_kind(greptimedb_connector_send_query_error, Trace) + ); + false -> + ?assertMatch( + [#{error := [{error, no_fields}]} | _], + ?of_kind(greptimedb_connector_send_query_error, Trace) + ) + end, + %% nothing should have been persisted + ?assertEqual(#{}, PersistedData0), + ?assertEqual(#{}, PersistedData1), + ok + end + ), + ok. + +t_authentication_error(Config0) -> + GreptimedbType = ?config(greptimedb_type, Config0), + GreptimeConfig0 = proplists:get_value(greptimedb_config, Config0), + GreptimeConfig = + case GreptimedbType of + grpcv1 -> GreptimeConfig0#{<<"password">> => <<"wrong_password">>} + end, + Config = lists:keyreplace(greptimedb_config, 1, Config0, {greptimedb_config, GreptimeConfig}), + ?check_trace( + begin + ?wait_async_action( + create_bridge(Config), + #{?snk_kind := greptimedb_connector_start_failed}, + 10_000 + ) + end, + fun(Trace) -> + ?assertMatch( + [#{error := auth_error} | _], + ?of_kind(greptimedb_connector_start_failed, Trace) + ), + ok + end + ), + ok. + +t_authentication_error_on_get_status(Config0) -> + ResourceId = resource_id(Config0), + + % Fake initialization to simulate credential update after bridge was created. + emqx_common_test_helpers:with_mock( + greptimedb, + check_auth, + fun(_) -> + ok + end, + fun() -> + GreptimedbType = ?config(greptimedb_type, Config0), + GreptimeConfig0 = proplists:get_value(greptimedb_config, Config0), + GreptimeConfig = + case GreptimedbType of + grpcv1 -> GreptimeConfig0#{<<"password">> => <<"wrong_password">>} + end, + Config = lists:keyreplace( + greptimedb_config, 1, Config0, {greptimedb_config, GreptimeConfig} + ), + {ok, _} = create_bridge(Config), + ?retry( + _Sleep = 1_000, + _Attempts = 10, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ) + end + ), + + % Now back to wrong credentials + ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceId)), + ok. + +t_authentication_error_on_send_message(Config0) -> + ResourceId = resource_id(Config0), + QueryMode = proplists:get_value(query_mode, Config0, sync), + GreptimedbType = ?config(greptimedb_type, Config0), + GreptimeConfig0 = proplists:get_value(greptimedb_config, Config0), + GreptimeConfig = + case GreptimedbType of + grpcv1 -> GreptimeConfig0#{<<"password">> => <<"wrong_password">>} + end, + Config = lists:keyreplace(greptimedb_config, 1, Config0, {greptimedb_config, GreptimeConfig}), + + % Fake initialization to simulate credential update after bridge was created. + emqx_common_test_helpers:with_mock( + greptimedb, + check_auth, + fun(_) -> + ok + end, + fun() -> + {ok, _} = create_bridge(Config), + ?retry( + _Sleep = 1_000, + _Attempts = 10, + ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) + ) + end + ), + + % Now back to wrong credentials + ClientId = emqx_guid:to_hexstr(emqx_guid:gen()), + Payload = #{ + int_key => -123, + bool => true, + float_key => 24.5, + uint_key => 123 + }, + SentData = #{ + <<"clientid">> => ClientId, + <<"topic">> => atom_to_binary(?FUNCTION_NAME), + <<"timestamp">> => erlang:system_time(millisecond), + <<"payload">> => Payload + }, + case QueryMode of + sync -> + ?assertMatch( + {error, {unrecoverable_error, <<"authorization failure">>}}, + send_message(Config, SentData) + ) + end, + ok. From 975795a6e0d21069d30d57b33144ab5c94b6ce3f Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 7 Jul 2023 19:27:21 +0800 Subject: [PATCH 07/21] feat: add ci test --- .../docker-compose-greptimedb.yaml | 15 ++++++ .../docker-compose-toxiproxy.yaml | 3 ++ .ci/docker-compose-file/toxiproxy.json | 11 ++++ .../src/schema/emqx_bridge_enterprise.erl | 8 --- .../src/emqx_bridge_greptimedb.erl | 1 + .../test/emqx_bridge_greptimedb_SUITE.erl | 9 ++-- rel/i18n/emqx_bridge_greptimedb.hocon | 50 +++++++++++++++++++ .../emqx_bridge_greptimedb_connector.hocon | 47 +++++++++++++++++ scripts/ct/run.sh | 3 ++ 9 files changed, 135 insertions(+), 12 deletions(-) create mode 100644 .ci/docker-compose-file/docker-compose-greptimedb.yaml create mode 100644 rel/i18n/emqx_bridge_greptimedb.hocon create mode 100644 rel/i18n/emqx_bridge_greptimedb_connector.hocon diff --git a/.ci/docker-compose-file/docker-compose-greptimedb.yaml b/.ci/docker-compose-file/docker-compose-greptimedb.yaml new file mode 100644 index 000000000..f379969bd --- /dev/null +++ b/.ci/docker-compose-file/docker-compose-greptimedb.yaml @@ -0,0 +1,15 @@ +version: '3.9' + +services: + greptimedb: + container_name: greptimedb + image: greptime/greptimedb:0.3.2 + expose: + - "4000" + - "4001" + restart: always + networks: + - emqx_bridge + command: + standalone start + --user-provider=static_user_provider:cmd:greptime_user=greptime_pwd diff --git a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml index 74d2583c9..d648d9d78 100644 --- a/.ci/docker-compose-file/docker-compose-toxiproxy.yaml +++ b/.ci/docker-compose-file/docker-compose-toxiproxy.yaml @@ -51,6 +51,9 @@ services: - 15670:5670 # Kinesis - 4566:4566 + # GreptimeDB + - 4000:4000 + - 4001:4001 command: - "-host=0.0.0.0" - "-config=/config/toxiproxy.json" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index c9590354b..a8e2f086c 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -160,6 +160,17 @@ "name": "hstreamdb", "listen": "0.0.0.0:6570", "upstream": "hstreamdb:6570", + }, + { + "name": "greptimedb_http", + "listen": "0.0.0.0:4000", + "upstream": "iotdb:4000", + "enabled": true + }, + { + "name": "greptimedb_grpc", + "listen": "0.0.0.0:4001", + "upstream": "iotdb:4001", "enabled": true }, { diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index 048dcbf90..4a0428675 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -200,14 +200,6 @@ fields(bridges) -> desc => <<"Apache IoTDB Bridge Config">>, required => false } - )}, - {greptimedb, - mk( - hoconsc:map(name, ref(emqx_bridge_greptimedb, "config")), - #{ - desc => <<"GreptimeDB Bridge Config">>, - required => false - } )} ] ++ kafka_structs() ++ pulsar_structs() ++ gcp_pubsub_structs() ++ mongodb_structs() ++ influxdb_structs() ++ diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl index 5bd8f6852..415544fcf 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -107,6 +107,7 @@ method_fields(put, ConnectorType) -> greptimedb_bridge_common_fields() -> emqx_bridge_schema:common_bridge_fields() ++ [ + {local_topic, mk(binary(), #{desc => ?DESC("local_topic")})}, {write_syntax, fun write_syntax/1} ] ++ emqx_resource_schema:fields("resource_opts"). diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index e694060f5..57ffed926 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -30,11 +30,12 @@ groups() -> {group, sync_query} ]}, {sync_query, [ - {group, grpcv1_tcp}, - {group, grpcv1_tls} + {group, grpcv1_tcp} + %% uncomment tls when we are ready + %% {group, grpcv1_tls} ]}, - {grpcv1_tcp, TCs}, - {grpcv1_tls, TCs} + {grpcv1_tcp, TCs} + %%{grpcv1_tls, TCs} ]. init_per_suite(Config) -> diff --git a/rel/i18n/emqx_bridge_greptimedb.hocon b/rel/i18n/emqx_bridge_greptimedb.hocon new file mode 100644 index 000000000..939ed48d3 --- /dev/null +++ b/rel/i18n/emqx_bridge_greptimedb.hocon @@ -0,0 +1,50 @@ +emqx_bridge_greptimedb { + +config_enable.desc: +"""Enable or disable this bridge.""" + +config_enable.label: +"""Enable Or Disable Bridge""" + +desc_config.desc: +"""Configuration for an GreptimeDB bridge.""" + +desc_config.label: +"""GreptimeDB Bridge Configuration""" + +desc_name.desc: +"""Bridge name.""" + +desc_name.label: +"""Bridge Name""" + +desc_type.desc: +"""The Bridge Type.""" + +desc_type.label: +"""Bridge Type""" + +local_topic.desc: +"""The MQTT topic filter to be forwarded to the GreptimeDB. All MQTT 'PUBLISH' messages with the topic +matching the local_topic will be forwarded.
+NOTE: if this bridge is used as the action of a rule (EMQX rule engine), and also local_topic is +configured, then both the data got from the rule and the MQTT messages that match local_topic +will be forwarded.""" + +local_topic.label: +"""Local Topic""" + +write_syntax.desc: +"""Conf of GreptimeDB gRPC protocol to write data points.The write syntax is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported, which is the same as InfluxDB line protocol. +See also [InfluxDB 2.3 Line Protocol](https://docs.influxdata.com/influxdb/v2.3/reference/syntax/line-protocol/) and +[GreptimeDB 1.8 Line Protocol](https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/)
+TLDR:
+``` +[,=[,=]] =[,=] [] +``` +Please note that a placeholder for an integer value must be annotated with a suffix `i`. For example `${payload.int_value}i`.""" + +write_syntax.label: +"""Write Syntax""" + +} diff --git a/rel/i18n/emqx_bridge_greptimedb_connector.hocon b/rel/i18n/emqx_bridge_greptimedb_connector.hocon new file mode 100644 index 000000000..87370b211 --- /dev/null +++ b/rel/i18n/emqx_bridge_greptimedb_connector.hocon @@ -0,0 +1,47 @@ +emqx_bridge_greptimedb_connector { + +dbname.desc: +"""GreptimeDB database.""" + +dbname.label: +"""Database""" + +greptimedb_grpc_v1.desc: +"""GreptimeDB's protocol. Support GreptimeDB v1.8 and before.""" + +greptimedb_grpc_v1.label: +"""HTTP API Protocol""" + +password.desc: +"""GreptimeDB password.""" + +password.label: +"""Password""" + +precision.desc: +"""GreptimeDB time precision.""" + +precision.label: +"""Time Precision""" + +protocol.desc: +"""GreptimeDB's protocol. gRPC API.""" + +protocol.label: +"""Protocol""" + +server.desc: +"""The IPv4 or IPv6 address or the hostname to connect to.
+A host entry has the following form: `Host[:Port]`.
+The GreptimeDB default port 8086 is used if `[:Port]` is not specified.""" + +server.label: +"""Server Host""" + +username.desc: +"""GreptimeDB username.""" + +username.label: +"""Username""" + +} diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index 785d4065d..578b9c4de 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -222,6 +222,9 @@ for dep in ${CT_DEPS}; do kinesis) FILES+=( '.ci/docker-compose-file/docker-compose-kinesis.yaml' ) ;; + greptimedb) + FILES+=( '.ci/docker-compose-file/docker-compose-greptimedb.yaml' ) + ;; *) echo "unknown_ct_dependency $dep" exit 1 From c6a7f3e2ade7769960b560ccd932593cca437c62 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Sun, 16 Jul 2023 17:46:04 +0800 Subject: [PATCH 08/21] test: make test passed 21/29 --- .../docker-compose-greptimedb.yaml | 7 + .ci/docker-compose-file/toxiproxy.json | 5 +- .../src/emqx_bridge_greptimedb_connector.erl | 2 +- .../test/emqx_bridge_greptimedb_SUITE.erl | 200 +++++------------- 4 files changed, 69 insertions(+), 145 deletions(-) diff --git a/.ci/docker-compose-file/docker-compose-greptimedb.yaml b/.ci/docker-compose-file/docker-compose-greptimedb.yaml index f379969bd..8980c946d 100644 --- a/.ci/docker-compose-file/docker-compose-greptimedb.yaml +++ b/.ci/docker-compose-file/docker-compose-greptimedb.yaml @@ -3,13 +3,20 @@ version: '3.9' services: greptimedb: container_name: greptimedb + hostname: greptimedb image: greptime/greptimedb:0.3.2 expose: - "4000" - "4001" + # uncomment for local testing + # ports: + # - "4000:4000" + # - "4001:4001" restart: always networks: - emqx_bridge command: standalone start --user-provider=static_user_provider:cmd:greptime_user=greptime_pwd + --http-addr="0.0.0.0:4000" + --rpc-addr="0.0.0.0:4001" diff --git a/.ci/docker-compose-file/toxiproxy.json b/.ci/docker-compose-file/toxiproxy.json index a8e2f086c..f5df5a853 100644 --- a/.ci/docker-compose-file/toxiproxy.json +++ b/.ci/docker-compose-file/toxiproxy.json @@ -160,17 +160,18 @@ "name": "hstreamdb", "listen": "0.0.0.0:6570", "upstream": "hstreamdb:6570", + "enabled": true }, { "name": "greptimedb_http", "listen": "0.0.0.0:4000", - "upstream": "iotdb:4000", + "upstream": "greptimedb:4000", "enabled": true }, { "name": "greptimedb_grpc", "listen": "0.0.0.0:4001", - "upstream": "iotdb:4001", + "upstream": "greptimedb:4001", "enabled": true }, { diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index bc4eacbab..43455d5d2 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -312,7 +312,7 @@ do_query(InstId, Client, Points) -> connector => InstId, points => Points }); - {error, {401, _, _}} -> + {error, {unauth, _, _}} -> ?tp(greptimedb_connector_do_query_failure, #{error => <<"authorization failure">>}), ?SLOG(error, #{ msg => "greptimedb_authorization_failed", diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index 57ffed926..044d6a2bd 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -57,6 +57,7 @@ init_per_group(GreptimedbType, Config0) when #{ host := GreptimedbHost, port := GreptimedbPort, + http_port := GreptimedbHttpPort, use_tls := UseTLS, proxy_name := ProxyName } = @@ -65,13 +66,15 @@ init_per_group(GreptimedbType, Config0) when #{ host => os:getenv("GREPTIMEDB_GRPCV1_TCP_HOST", "toxiproxy"), port => list_to_integer(os:getenv("GREPTIMEDB_GRPCV1_TCP_PORT", "4001")), + http_port => list_to_integer(os:getenv("GREPTIMEDB_HTTP_PORT", "4000")), use_tls => false, - proxy_name => "greptimedb_tcp" + proxy_name => "greptimedb_grpc" }; grpcv1_tls -> #{ host => os:getenv("GREPTIMEDB_GRPCV1_TLS_HOST", "toxiproxy"), port => list_to_integer(os:getenv("GREPTIMEDB_GRPCV1_TLS_PORT", "4001")), + http_port => list_to_integer(os:getenv("GREPTIMEDB_HTTP_PORT", "4000")), use_tls => true, proxy_name => "greptimedb_tls" } @@ -98,7 +101,7 @@ init_per_group(GreptimedbType, Config0) when end, EHttpcPoolOpts = [ {host, GreptimedbHost}, - {port, GreptimedbPort}, + {port, GreptimedbHttpPort}, {pool_size, 1}, {transport, EHttpcTransport}, {transport_opts, EHttpcTransportOpts} @@ -110,6 +113,7 @@ init_per_group(GreptimedbType, Config0) when {proxy_name, ProxyName}, {greptimedb_host, GreptimedbHost}, {greptimedb_port, GreptimedbPort}, + {greptimedb_http_port, GreptimedbHttpPort}, {greptimedb_type, grpcv1}, {greptimedb_config, GreptimedbConfig}, {greptimedb_config_string, ConfigString}, @@ -282,12 +286,12 @@ send_message(Config, Payload) -> BridgeId = emqx_bridge_resource:bridge_id(Type, Name), emqx_bridge:send_message(BridgeId, Payload). -query_by_clientid(ClientId, Config) -> +query_by_clientid(Topic, ClientId, Config) -> GreptimedbHost = ?config(greptimedb_host, Config), - GreptimedbPort = ?config(greptimedb_port, Config), + GreptimedbPort = ?config(greptimedb_http_port, Config), EHttpcPoolName = ?config(ehttpc_pool_name, Config), UseTLS = ?config(use_tls, Config), - Path = <<"/api/v2/query?org=emqx">>, + Path = <<"/v1/sql?db=public">>, Scheme = case UseTLS of true -> <<"https://">>; @@ -300,26 +304,11 @@ query_by_clientid(ClientId, Config) -> integer_to_binary(GreptimedbPort), Path ]), - Query = - << - "from(bucket: \"mqtt\")\n" - " |> range(start: -12h)\n" - " |> filter(fn: (r) => r.clientid == \"", - ClientId/binary, - "\")" - >>, Headers = [ - {"Authorization", "Token abcdefg"}, - {"Content-Type", "application/json"} + {"Authorization", "Basic Z3JlcHRpbWVfdXNlcjpncmVwdGltZV9wd2Q="}, + {"Content-Type", "application/x-www-form-urlencoded"} ], - Body = - emqx_utils_json:encode(#{ - query => Query, - dialect => #{ - header => true, - delimiter => <<";">> - } - }), + Body = <<"sql=select * from ", Topic/binary, " where clientid='", ClientId/binary, "'">>, {ok, 200, _Headers, RawBody0} = ehttpc:request( EHttpcPoolName, @@ -328,32 +317,30 @@ query_by_clientid(ClientId, Config) -> _Timeout = 10_000, _Retry = 0 ), - RawBody1 = iolist_to_binary(string:replace(RawBody0, <<"\r\n">>, <<"\n">>, all)), - {ok, DecodedCSV0} = erl_csv:decode(RawBody1, #{separator => <<$;>>}), - DecodedCSV1 = [ - [Field || Field <- Line, Field =/= <<>>] - || Line <- DecodedCSV0, - Line =/= [<<>>] - ], - DecodedCSV2 = csv_lines_to_maps(DecodedCSV1, []), - index_by_field(DecodedCSV2). + #{ + <<"code">> := 0, + <<"output">> := [ + #{ + <<"records">> := #{ + <<"rows">> := Rows, + <<"schema">> := Schema + } + } + ] + } = emqx_utils_json:decode(RawBody0, [return_maps]), -decode_csv(RawBody) -> - Lines = - [ - binary:split(Line, [<<";">>], [global, trim_all]) - || Line <- binary:split(RawBody, [<<"\r\n">>], [global, trim_all]) - ], - csv_lines_to_maps(Lines, []). + case Schema of + null -> + #{}; + #{<<"column_schemas">> := ColumnsSchemas} -> + Columns = lists:map(fun(#{<<"name">> := Name}) -> Name end, ColumnsSchemas), + index_by_field(Rows, Columns) + end. -csv_lines_to_maps([Fields, Data | Rest], Acc) -> - Map = maps:from_list(lists:zip(Fields, Data)), - csv_lines_to_maps(Rest, [Map | Acc]); -csv_lines_to_maps(_Data, Acc) -> - lists:reverse(Acc). - -index_by_field(DecodedCSV) -> - maps:from_list([{Field, Data} || Data = #{<<"_field">> := Field} <- DecodedCSV]). +index_by_field([], Columns) -> + #{}; +index_by_field([Row], Columns) -> + maps:from_list(lists:zip(Columns, Row)). assert_persisted_data(ClientId, Expected, PersistedData) -> ClientIdIntKey = <>, @@ -361,12 +348,12 @@ assert_persisted_data(ClientId, Expected, PersistedData) -> fun (int_value, ExpectedValue) -> ?assertMatch( - #{<<"_value">> := ExpectedValue}, + ExpectedValue, maps:get(ClientIdIntKey, PersistedData) ); (Key, ExpectedValue) -> ?assertMatch( - #{<<"_value">> := ExpectedValue}, + ExpectedValue, maps:get(atom_to_binary(Key), PersistedData), #{expected => ExpectedValue} ) @@ -409,12 +396,12 @@ t_start_ok(Config) -> sync -> ?assertMatch(ok, send_message(Config, SentData)) end, - PersistedData = query_by_clientid(ClientId, Config), + PersistedData = query_by_clientid(atom_to_binary(?FUNCTION_NAME), ClientId, Config), Expected = #{ - bool => <<"true">>, - int_value => <<"-123">>, - uint_value => <<"123">>, - float_value => <<"24.5">>, + bool => true, + int_value => -123, + uint_value => 123, + float_value => 24.5, payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), @@ -423,12 +410,16 @@ t_start_ok(Config) -> fun(Trace0) -> Trace = ?of_kind(greptimedb_connector_send_query, Trace0), ?assertMatch([#{points := [_]}], Trace), - [#{points := [Point]}] = Trace, + [#{points := [Point0]}] = Trace, + {Measurement, [Point]} = Point0, ct:pal("sent point: ~p", [Point]), + ?assertMatch( + <<_/binary>>, + Measurement + ), ?assertMatch( #{ fields := #{}, - measurement := <<_/binary>>, tags := #{}, timestamp := TS } when is_integer(TS), @@ -538,9 +529,6 @@ t_const_timestamp(Config) -> QueryMode = ?config(query_mode, Config), Const = erlang:system_time(nanosecond), ConstBin = integer_to_binary(Const), - TsStr = iolist_to_binary( - calendar:system_time_to_rfc3339(Const, [{unit, nanosecond}, {offset, "Z"}]) - ), ?assertMatch( {ok, _}, create_bridge( @@ -563,24 +551,11 @@ t_const_timestamp(Config) -> sync -> ?assertMatch(ok, send_message(Config, SentData)) end, - PersistedData = query_by_clientid(ClientId, Config), - Expected = #{foo => <<"123">>}, + PersistedData = query_by_clientid(<<"mqtt">>, ClientId, Config), + Expected = #{foo => 123}, assert_persisted_data(ClientId, Expected, PersistedData), - TimeReturned0 = maps:get(<<"_time">>, maps:get(<<"foo">>, PersistedData)), - TimeReturned = pad_zero(TimeReturned0), - ?assertEqual(TsStr, TimeReturned). - -%% greptimedb returns timestamps without trailing zeros such as -%% "2023-02-28T17:21:51.63678163Z" -%% while the standard should be -%% "2023-02-28T17:21:51.636781630Z" -pad_zero(BinTs) -> - StrTs = binary_to_list(BinTs), - [Nano | Rest] = lists:reverse(string:tokens(StrTs, ".")), - [$Z | NanoNum] = lists:reverse(Nano), - Padding = lists:duplicate(10 - length(Nano), $0), - NewNano = lists:reverse(NanoNum) ++ Padding ++ "Z", - iolist_to_binary(string:join(lists:reverse([NewNano | Rest]), ".")). + TimeReturned = maps:get(<<"greptime_timestamp">>, PersistedData), + ?assertEqual(Const, TimeReturned). t_boolean_variants(Config) -> QueryMode = ?config(query_mode, Config), @@ -621,11 +596,11 @@ t_boolean_variants(Config) -> case QueryMode of sync -> ok end, - PersistedData = query_by_clientid(ClientId, Config), + PersistedData = query_by_clientid(atom_to_binary(?FUNCTION_NAME), ClientId, Config), Expected = #{ - bool => atom_to_binary(Translation), - int_value => <<"-123">>, - uint_value => <<"123">>, + bool => Translation, + int_value => -123, + uint_value => 123, payload => emqx_utils_json:encode(Payload) }, assert_persisted_data(ClientId, Expected, PersistedData), @@ -728,7 +703,7 @@ t_create_disconnected(Config) -> end), fun(Trace) -> ?assertMatch( - [#{error := greptimedb_client_not_alive, reason := econnrefused}], + [#{error := greptimedb_client_not_alive, reason := _SomeReason}], ?of_kind(greptimedb_connector_start_failed, Trace) ), ok @@ -871,8 +846,8 @@ t_missing_field(Config) -> ok end, fun(Trace) -> - PersistedData0 = query_by_clientid(ClientId0, Config), - PersistedData1 = query_by_clientid(ClientId1, Config), + PersistedData0 = query_by_clientid(ClientId0, ClientId0, Config), + PersistedData1 = query_by_clientid(ClientId1, ClientId1, Config), case IsBatch of true -> ?assertMatch( @@ -893,65 +868,6 @@ t_missing_field(Config) -> ), ok. -t_authentication_error(Config0) -> - GreptimedbType = ?config(greptimedb_type, Config0), - GreptimeConfig0 = proplists:get_value(greptimedb_config, Config0), - GreptimeConfig = - case GreptimedbType of - grpcv1 -> GreptimeConfig0#{<<"password">> => <<"wrong_password">>} - end, - Config = lists:keyreplace(greptimedb_config, 1, Config0, {greptimedb_config, GreptimeConfig}), - ?check_trace( - begin - ?wait_async_action( - create_bridge(Config), - #{?snk_kind := greptimedb_connector_start_failed}, - 10_000 - ) - end, - fun(Trace) -> - ?assertMatch( - [#{error := auth_error} | _], - ?of_kind(greptimedb_connector_start_failed, Trace) - ), - ok - end - ), - ok. - -t_authentication_error_on_get_status(Config0) -> - ResourceId = resource_id(Config0), - - % Fake initialization to simulate credential update after bridge was created. - emqx_common_test_helpers:with_mock( - greptimedb, - check_auth, - fun(_) -> - ok - end, - fun() -> - GreptimedbType = ?config(greptimedb_type, Config0), - GreptimeConfig0 = proplists:get_value(greptimedb_config, Config0), - GreptimeConfig = - case GreptimedbType of - grpcv1 -> GreptimeConfig0#{<<"password">> => <<"wrong_password">>} - end, - Config = lists:keyreplace( - greptimedb_config, 1, Config0, {greptimedb_config, GreptimeConfig} - ), - {ok, _} = create_bridge(Config), - ?retry( - _Sleep = 1_000, - _Attempts = 10, - ?assertEqual({ok, connected}, emqx_resource_manager:health_check(ResourceId)) - ) - end - ), - - % Now back to wrong credentials - ?assertEqual({ok, disconnected}, emqx_resource_manager:health_check(ResourceId)), - ok. - t_authentication_error_on_send_message(Config0) -> ResourceId = resource_id(Config0), QueryMode = proplists:get_value(query_mode, Config0, sync), From 49218569503e6502edd6a180a2f170626efd9304 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Thu, 20 Jul 2023 20:10:29 +0800 Subject: [PATCH 09/21] test: make all emqx_bridge_greptimedb_SUITE tests passing --- .../src/emqx_bridge_greptimedb_connector.erl | 4 +- .../test/emqx_bridge_greptimedb_SUITE.erl | 94 +++++++++++-------- 2 files changed, 58 insertions(+), 40 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 43455d5d2..655351842 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -81,7 +81,7 @@ on_query(InstId, {send_message, Data}, _State = #{write_syntax := SyntaxLines, c #{batch => false, mode => sync, error => ErrorPoints} ), log_error_points(InstId, ErrorPoints), - ErrorPoints + {error, ErrorPoints} end. %% Once a Batched Data trans to points failed. @@ -463,7 +463,7 @@ parse_timestamp([TsBin]) -> continue_lines_to_points(Data, Item, Rest, ResultPointsAcc, ErrorPointsAcc) -> case line_to_point(Data, Item) of - #{fields := Fields} when map_size(Fields) =:= 0 -> + {_, [#{fields := Fields}]} when map_size(Fields) =:= 0 -> %% greptimedb client doesn't like empty field maps... ErrorPointsAcc1 = [{error, no_fields} | ErrorPointsAcc], lines_to_points(Data, Rest, ResultPointsAcc, ErrorPointsAcc1); diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index 044d6a2bd..b7fb6451e 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -9,6 +9,7 @@ -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). +-include_lib("emqx/include/logger.hrl"). %%------------------------------------------------------------------------------ %% CT boilerplate @@ -284,7 +285,8 @@ send_message(Config, Payload) -> Name = ?config(greptimedb_name, Config), Type = greptimedb_type_bin(?config(greptimedb_type, Config)), BridgeId = emqx_bridge_resource:bridge_id(Type, Name), - emqx_bridge:send_message(BridgeId, Payload). + Resp = emqx_bridge:send_message(BridgeId, Payload), + Resp. query_by_clientid(Topic, ClientId, Config) -> GreptimedbHost = ?config(greptimedb_host, Config), @@ -308,7 +310,7 @@ query_by_clientid(Topic, ClientId, Config) -> {"Authorization", "Basic Z3JlcHRpbWVfdXNlcjpncmVwdGltZV9wd2Q="}, {"Content-Type", "application/x-www-form-urlencoded"} ], - Body = <<"sql=select * from ", Topic/binary, " where clientid='", ClientId/binary, "'">>, + Body = <<"sql=select * from \"", Topic/binary, "\" where clientid='", ClientId/binary, "'">>, {ok, 200, _Headers, RawBody0} = ehttpc:request( EHttpcPoolName, @@ -317,29 +319,49 @@ query_by_clientid(Topic, ClientId, Config) -> _Timeout = 10_000, _Retry = 0 ), - #{ - <<"code">> := 0, - <<"output">> := [ - #{ - <<"records">> := #{ - <<"rows">> := Rows, - <<"schema">> := Schema - } - } - ] - } = emqx_utils_json:decode(RawBody0, [return_maps]), - case Schema of - null -> - #{}; - #{<<"column_schemas">> := ColumnsSchemas} -> - Columns = lists:map(fun(#{<<"name">> := Name}) -> Name end, ColumnsSchemas), - index_by_field(Rows, Columns) + case emqx_utils_json:decode(RawBody0, [return_maps]) of + #{ + <<"code">> := 0, + <<"output">> := [ + #{ + <<"records">> := #{ + <<"rows">> := Rows, + <<"schema">> := Schema + } + } + ] + } -> + make_row(Schema, Rows); + #{ + <<"code">> := Code, + <<"error">> := Error + } -> + GreptimedbName = ?config(greptimedb_name, Config), + Type = greptimedb_type_bin(?config(greptimedb_type, Config)), + BridgeId = emqx_bridge_resource:bridge_id(Type, GreptimedbName), + + ?SLOG(error, #{ + msg => io_lib:format("Failed to query: ~p, ~p", [Code, Error]), + connector => BridgeId, + reason => Error + }), + %% TODO(dennis): check the error by code + case binary:match(Error, <<"Table not found">>) of + nomatch -> + {error, Error}; + _ -> + %% Table not found + #{} + end end. -index_by_field([], Columns) -> +make_row(null, _Rows) -> #{}; -index_by_field([Row], Columns) -> +make_row(_Schema, []) -> + #{}; +make_row(#{<<"column_schemas">> := ColumnsSchemas}, [Row]) -> + Columns = lists:map(fun(#{<<"name">> := Name}) -> Name end, ColumnsSchemas), maps:from_list(lists:zip(Columns, Row)). assert_persisted_data(ClientId, Expected, PersistedData) -> @@ -784,26 +806,22 @@ t_write_failure(Config) -> emqx_common_test_helpers:with_failure(down, ProxyName, ProxyHost, ProxyPort, fun() -> case QueryMode of sync -> - {_, {ok, _}} = - ?wait_async_action( - ?assertMatch( - {error, {resource_error, #{reason := timeout}}}, - send_message(Config, SentData) - ), - #{?snk_kind := handle_async_reply, action := nack}, - 1_000 - ) + ?wait_async_action( + ?assertMatch( + {error, {resource_error, #{reason := timeout}}}, + send_message(Config, SentData) + ), + #{?snk_kind := greptimedb_connector_do_query_failure, action := nack}, + 16_000 + ) end end), - fun(Trace0) -> + fun(Trace) -> case QueryMode of sync -> - Trace = ?of_kind(handle_async_reply, Trace0), - ?assertMatch([_ | _], Trace), - [#{result := Result} | _] = Trace, - ?assert( - not emqx_bridge_greptimedb_connector:is_unrecoverable_error(Result), - #{got => Result} + ?assertMatch( + [#{error := _} | _], + ?of_kind(greptimedb_connector_do_query_failure, Trace) ) end, ok @@ -841,7 +859,7 @@ t_missing_field(Config) -> ?match_n_events(NEvents, #{ ?snk_kind := greptimedb_connector_send_query_error }), - _Timeout1 = 10_000 + _Timeout1 = 16_000 ), ok end, From 6f7fbcf6937790b507ea7bb931dcef33103fd746 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Thu, 20 Jul 2023 20:17:07 +0800 Subject: [PATCH 10/21] fix: compile error --- apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index 4a0428675..c6765e859 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -123,7 +123,7 @@ resource_type(pulsar_producer) -> emqx_bridge_pulsar_impl_producer; resource_type(oracle) -> emqx_oracle; resource_type(iotdb) -> emqx_bridge_iotdb_impl; resource_type(rabbitmq) -> emqx_bridge_rabbitmq_connector; -resource_type(kinesis_producer) -> emqx_bridge_kinesis_impl_producer. +resource_type(kinesis_producer) -> emqx_bridge_kinesis_impl_producer; resource_type(greptimedb_grpc_v1) -> emqx_bridge_greptimedb_connector. fields(bridges) -> From 3b1363dbb7751008ed9199de14719a5cd552d0b8 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Thu, 20 Jul 2023 20:51:52 +0800 Subject: [PATCH 11/21] chore: change license to BCL and adds emqx_bridge_greptimedb_connector_SUITE --- apps/emqx_bridge_greptimedb/BSL.txt | 94 +++++++++ apps/emqx_bridge_greptimedb/LICENSE | 191 ------------------ apps/emqx_bridge_greptimedb/README.md | 14 +- .../src/emqx_bridge_greptimedb.app.src | 3 +- .../src/emqx_bridge_greptimedb_connector.erl | 10 +- .../test/emqx_bridge_greptimedb_SUITE.erl | 7 +- ...emqx_bridge_greptimedb_connector_SUITE.erl | 152 ++++++++++++++ 7 files changed, 266 insertions(+), 205 deletions(-) create mode 100644 apps/emqx_bridge_greptimedb/BSL.txt delete mode 100644 apps/emqx_bridge_greptimedb/LICENSE create mode 100644 apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl diff --git a/apps/emqx_bridge_greptimedb/BSL.txt b/apps/emqx_bridge_greptimedb/BSL.txt new file mode 100644 index 000000000..0acc0e696 --- /dev/null +++ b/apps/emqx_bridge_greptimedb/BSL.txt @@ -0,0 +1,94 @@ +Business Source License 1.1 + +Licensor: Hangzhou EMQ Technologies Co., Ltd. +Licensed Work: EMQX Enterprise Edition + The Licensed Work is (c) 2023 + Hangzhou EMQ Technologies Co., Ltd. +Additional Use Grant: Students and educators are granted right to copy, + modify, and create derivative work for research + or education. +Change Date: 2027-02-01 +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please contact Licensor: https://www.emqx.com/en/contact + +Notice + +The Business Source License (this document, or the “License”) is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +“Business Source License” is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark “Business Source License”, +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the “Business +Source License” name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where “compatible” means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text “None”. + +3. To specify a Change Date. + +4. Not to modify this License in any other way. diff --git a/apps/emqx_bridge_greptimedb/LICENSE b/apps/emqx_bridge_greptimedb/LICENSE deleted file mode 100644 index 64d3c22a9..000000000 --- a/apps/emqx_bridge_greptimedb/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2023, Dennis Zhuang . - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - diff --git a/apps/emqx_bridge_greptimedb/README.md b/apps/emqx_bridge_greptimedb/README.md index 13f26c348..b92538c66 100644 --- a/apps/emqx_bridge_greptimedb/README.md +++ b/apps/emqx_bridge_greptimedb/README.md @@ -13,7 +13,15 @@ For more information about GreptimeDB, please refer to [official ## Configurations -TODO +Just like the InfluxDB data bridge but have some different parameters. Below are several important parameters: + - `server`: The IPv4 or IPv6 address or the hostname to connect to. + - `dbname`: The GreptimeDB database name. + - `write_syntax`: Like the `write_syntax` in `InfluxDB` conf, it's the conf of InfluxDB line protocol to write data points. It is a text-based format that provides the measurement, tag set, field set, and timestamp of a data point, and placeholder supported. -## License -[Apache License 2.0](./LICENSE) + +# Contributing - [Mandatory] +Please see our [contributing.md](../../CONTRIBUTING.md). + +# License + +See [BSL](./BSL.txt). diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index f0a07bc28..53053c80b 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -5,10 +5,11 @@ {applications, [ kernel, stdlib, + emqx_resource, + emqx_bridge, greptimedb ]}, {env, []}, {modules, []}, - {licenses, ["Apache-2.0"]}, {links, []} ]}. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 655351842..a31606bbc 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -129,10 +129,7 @@ fields(common) -> [ {server, server()}, {precision, - %% The greptimedb only supports these 4 precision: - %% See "https://github.com/influxdata/greptimedb/blob/ - %% 6b607288439a991261307518913eb6d4e280e0a7/models/points.go#L487" for - %% more information. + %% The greptimedb only supports these 4 precision mk(enum([ns, us, ms, s]), #{ required => false, default => ms, desc => ?DESC("precision") })} @@ -306,12 +303,13 @@ is_auth_key(_) -> %% Query do_query(InstId, Client, Points) -> case greptimedb:write_batch(Client, Points) of - {ok, _} -> + {ok, #{response := {affected_rows, #{value := Rows}}}} -> ?SLOG(debug, #{ msg => "greptimedb write point success", connector => InstId, points => Points - }); + }), + {ok, {affected_rows, Rows}}; {error, {unauth, _, _}} -> ?tp(greptimedb_connector_do_query_failure, #{error => <<"authorization failure">>}), ?SLOG(error, #{ diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index b7fb6451e..1a38c882f 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -87,7 +87,6 @@ init_per_group(GreptimedbType, Config0) when emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), ok = start_apps(), {ok, _} = application:ensure_all_started(emqx_connector), - application:ensure_all_started(greptimedb), emqx_mgmt_api_test_util:init_suite(), Config = [{use_tls, UseTLS} | Config0], {Name, ConfigString, GreptimedbConfig} = greptimedb_config( @@ -416,7 +415,7 @@ t_start_ok(Config) -> begin case QueryMode of sync -> - ?assertMatch(ok, send_message(Config, SentData)) + ?assertMatch({ok, _}, send_message(Config, SentData)) end, PersistedData = query_by_clientid(atom_to_binary(?FUNCTION_NAME), ClientId, Config), Expected = #{ @@ -571,7 +570,7 @@ t_const_timestamp(Config) -> }, case QueryMode of sync -> - ?assertMatch(ok, send_message(Config, SentData)) + ?assertMatch({ok, _}, send_message(Config, SentData)) end, PersistedData = query_by_clientid(<<"mqtt">>, ClientId, Config), Expected = #{foo => 123}, @@ -613,7 +612,7 @@ t_boolean_variants(Config) -> }, case QueryMode of sync -> - ?assertMatch(ok, send_message(Config, SentData)) + ?assertMatch({ok, _}, send_message(Config, SentData)) end, case QueryMode of sync -> ok diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl new file mode 100644 index 000000000..3e576393a --- /dev/null +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl @@ -0,0 +1,152 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved. +%%-------------------------------------------------------------------- + +-module(emqx_bridge_greptimedb_connector_SUITE). + +-compile(nowarn_export_all). +-compile(export_all). + +-include_lib("emqx_connector/include/emqx_connector.hrl"). +-include_lib("eunit/include/eunit.hrl"). +-include_lib("common_test/include/ct.hrl"). + +-define(GREPTIMEDB_RESOURCE_MOD, emqx_bridge_greptimedb_connector). + +all() -> + emqx_common_test_helpers:all(?MODULE). + +groups() -> + []. + +init_per_suite(Config) -> + GreptimedbTCPHost = os:getenv("GREPTIMEDB_GRPCV1_TCP_HOST", "toxiproxy"), + GreptimedbTCPPort = list_to_integer(os:getenv("GREPTIMEDB_GRPCV1_TCP_PORT", "4001")), + Servers = [{GreptimedbTCPHost, GreptimedbTCPPort}], + case emqx_common_test_helpers:is_all_tcp_servers_available(Servers) of + true -> + ok = emqx_common_test_helpers:start_apps([emqx_conf]), + ok = emqx_connector_test_helpers:start_apps([emqx_resource]), + {ok, _} = application:ensure_all_started(emqx_connector), + [ + {greptimedb_tcp_host, GreptimedbTCPHost}, + {greptimedb_tcp_port, GreptimedbTCPPort} + | Config + ]; + false -> + case os:getenv("IS_CI") of + "yes" -> + throw(no_greptimedb); + _ -> + {skip, no_greptimedb} + end + end. + +end_per_suite(_Config) -> + ok = emqx_common_test_helpers:stop_apps([emqx_conf]), + ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), + _ = application:stop(emqx_connector). + +init_per_testcase(_, Config) -> + Config. + +end_per_testcase(_, _Config) -> + ok. + +% %%------------------------------------------------------------------------------ +% %% Testcases +% %%------------------------------------------------------------------------------ + +t_lifecycle(Config) -> + Host = ?config(greptimedb_tcp_host, Config), + Port = ?config(greptimedb_tcp_port, Config), + perform_lifecycle_check( + <<"emqx_bridge_greptimedb_connector_SUITE">>, + greptimedb_config(Host, Port) + ). + +perform_lifecycle_check(PoolName, InitialConfig) -> + {ok, #{config := CheckedConfig}} = + emqx_resource:check_config(?GREPTIMEDB_RESOURCE_MOD, InitialConfig), + % We need to add a write_syntax to the config since the connector + % expects this + FullConfig = CheckedConfig#{write_syntax => greptimedb_write_syntax()}, + {ok, #{ + state := #{client := #{pool := ReturnedPoolName}} = State, + status := InitialStatus + }} = emqx_resource:create_local( + PoolName, + ?CONNECTOR_RESOURCE_GROUP, + ?GREPTIMEDB_RESOURCE_MOD, + FullConfig, + #{} + ), + ?assertEqual(InitialStatus, connected), + % Instance should match the state and status of the just started resource + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := InitialStatus + }} = + emqx_resource:get_instance(PoolName), + ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + % % Perform query as further check that the resource is working as expected + ?assertMatch({ok, _}, emqx_resource:query(PoolName, test_query())), + ?assertEqual(ok, emqx_resource:stop(PoolName)), + % Resource will be listed still, but state will be changed and healthcheck will fail + % as the worker no longer exists. + {ok, ?CONNECTOR_RESOURCE_GROUP, #{ + state := State, + status := StoppedStatus + }} = + emqx_resource:get_instance(PoolName), + ?assertEqual(stopped, StoppedStatus), + ?assertEqual({error, resource_is_stopped}, emqx_resource:health_check(PoolName)), + % Resource healthcheck shortcuts things by checking ets. Go deeper by checking pool itself. + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + % Can call stop/1 again on an already stopped instance + ?assertEqual(ok, emqx_resource:stop(PoolName)), + % Make sure it can be restarted and the healthchecks and queries work properly + ?assertEqual(ok, emqx_resource:restart(PoolName)), + % async restart, need to wait resource + timer:sleep(500), + {ok, ?CONNECTOR_RESOURCE_GROUP, #{status := InitialStatus}} = + emqx_resource:get_instance(PoolName), + ?assertEqual({ok, connected}, emqx_resource:health_check(PoolName)), + ?assertMatch({ok, _}, emqx_resource:query(PoolName, test_query())), + % Stop and remove the resource in one go. + ?assertEqual(ok, emqx_resource:remove_local(PoolName)), + ?assertEqual({error, not_found}, ecpool:stop_sup_pool(ReturnedPoolName)), + % Should not even be able to get the resource data out of ets now unlike just stopping. + ?assertEqual({error, not_found}, emqx_resource:get_instance(PoolName)). + +% %%------------------------------------------------------------------------------ +% %% Helpers +% %%------------------------------------------------------------------------------ + +greptimedb_config(Host, Port) -> + Server = list_to_binary(io_lib:format("~s:~b", [Host, Port])), + ResourceConfig = #{ + <<"dbname">> => <<"public">>, + <<"server">> => Server, + <<"username">> => <<"greptime_user">>, + <<"password">> => <<"greptime_pwd">> + }, + #{<<"config">> => ResourceConfig}. + +greptimedb_write_syntax() -> + [ + #{ + measurement => "${topic}", + tags => [{"clientid", "${clientid}"}], + fields => [{"payload", "${payload}"}], + timestamp => undefined + } + ]. + +test_query() -> + {send_message, #{ + <<"clientid">> => <<"something">>, + <<"payload">> => #{bool => true}, + <<"topic">> => <<"connector_test">>, + <<"timestamp">> => 1678220316257 + }}. From 50c10dd91971a2945cf5d6fe5b2a2b9ca6c83f52 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Thu, 20 Jul 2023 23:06:23 +0800 Subject: [PATCH 12/21] chore: update greptimedb-client-erl to v0.1.2 --- apps/emqx_bridge_greptimedb/rebar.config | 2 +- .../test/emqx_bridge_greptimedb_SUITE.erl | 2 ++ .../test/emqx_bridge_greptimedb_connector_SUITE.erl | 5 ++++- mix.exs | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/rebar.config b/apps/emqx_bridge_greptimedb/rebar.config index 2001a72fc..57d45997f 100644 --- a/apps/emqx_bridge_greptimedb/rebar.config +++ b/apps/emqx_bridge_greptimedb/rebar.config @@ -6,7 +6,7 @@ {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}}, - {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {branch, "feature/check-auth"}}} + {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.2"}}} ]}. {plugins, [rebar3_path_deps]}. {project_plugins, [erlfmt]}. diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index 1a38c882f..f563b7e71 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -49,6 +49,7 @@ end_per_suite(_Config) -> emqx_conf, emqx_bridge, emqx_resource, emqx_rule_engine ]), _ = application:stop(emqx_connector), + _ = application:stop(greptimedb), ok. init_per_group(GreptimedbType, Config0) when @@ -87,6 +88,7 @@ init_per_group(GreptimedbType, Config0) when emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), ok = start_apps(), {ok, _} = application:ensure_all_started(emqx_connector), + {ok, _} = application:ensure_all_started(greptimedb), emqx_mgmt_api_test_util:init_suite(), Config = [{use_tls, UseTLS} | Config0], {Name, ConfigString, GreptimedbConfig} = greptimedb_config( diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl index 3e576393a..a4acf5b4e 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_connector_SUITE.erl @@ -28,6 +28,7 @@ init_per_suite(Config) -> ok = emqx_common_test_helpers:start_apps([emqx_conf]), ok = emqx_connector_test_helpers:start_apps([emqx_resource]), {ok, _} = application:ensure_all_started(emqx_connector), + {ok, _} = application:ensure_all_started(greptimedb), [ {greptimedb_tcp_host, GreptimedbTCPHost}, {greptimedb_tcp_port, GreptimedbTCPPort} @@ -45,7 +46,9 @@ init_per_suite(Config) -> end_per_suite(_Config) -> ok = emqx_common_test_helpers:stop_apps([emqx_conf]), ok = emqx_connector_test_helpers:stop_apps([emqx_resource]), - _ = application:stop(emqx_connector). + _ = application:stop(emqx_connector), + _ = application:stop(greptimedb), + ok. init_per_testcase(_, Config) -> Config. diff --git a/mix.exs b/mix.exs index 4d6cf700b..21a238f22 100644 --- a/mix.exs +++ b/mix.exs @@ -209,6 +209,7 @@ defmodule EMQXUmbrella.MixProject do {:crc32cer, "0.1.8", override: true}, {:supervisor3, "1.1.12", override: true}, {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true}, + {:greptimedb, github: "GreptimeTeam/greptimedb-client-erl", tag: "v0.1.2", override: true}, # The following two are dependencies of rabbit_common. They are needed here to # make mix not complain about conflicting versions {:thoas, github: "emqx/thoas", tag: "v1.0.0", override: true}, From a1c7eb337be73c77030c8c5fef4f6175eeec88a0 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 21 Jul 2023 10:37:59 +0800 Subject: [PATCH 13/21] fix: dependency name --- apps/emqx_bridge_greptimedb/rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/emqx_bridge_greptimedb/rebar.config b/apps/emqx_bridge_greptimedb/rebar.config index 57d45997f..f0942f910 100644 --- a/apps/emqx_bridge_greptimedb/rebar.config +++ b/apps/emqx_bridge_greptimedb/rebar.config @@ -6,7 +6,7 @@ {emqx_connector, {path, "../../apps/emqx_connector"}}, {emqx_resource, {path, "../../apps/emqx_resource"}}, {emqx_bridge, {path, "../../apps/emqx_bridge"}}, - {greptimedb_client_erl, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.2"}}} + {greptimedb, {git, "https://github.com/GreptimeTeam/greptimedb-client-erl", {tag, "v0.1.2"}}} ]}. {plugins, [rebar3_path_deps]}. {project_plugins, [erlfmt]}. From ffcd04bc9fbb1d780964ee1fcda0d03f8afcdc2a Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 21 Jul 2023 10:42:50 +0800 Subject: [PATCH 14/21] docs: add change log --- changes/ee/feat-10647.en.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/ee/feat-10647.en.md diff --git a/changes/ee/feat-10647.en.md b/changes/ee/feat-10647.en.md new file mode 100644 index 000000000..2b341fa4b --- /dev/null +++ b/changes/ee/feat-10647.en.md @@ -0,0 +1 @@ +Add enterprise data bridge for [GreptimeDB](https://github.com/GreptimeTeam/greptimedb). \ No newline at end of file From c9550cc2e5a6b5925327e90d7c2b664aca0ad086 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 21 Jul 2023 14:33:05 +0800 Subject: [PATCH 15/21] refactor: rename bridge greptimedb_grpc_v1 to greptimedb --- apps/emqx_bridge/src/emqx_bridge.erl | 2 +- .../src/schema/emqx_bridge_enterprise.erl | 6 +++--- .../src/emqx_bridge_greptimedb.erl | 20 +++++++++---------- .../src/emqx_bridge_greptimedb_connector.erl | 10 +++++----- .../test/emqx_bridge_greptimedb_SUITE.erl | 10 +++++----- .../emqx_bridge_greptimedb_connector.hocon | 4 ++-- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/apps/emqx_bridge/src/emqx_bridge.erl b/apps/emqx_bridge/src/emqx_bridge.erl index b60276910..612481663 100644 --- a/apps/emqx_bridge/src/emqx_bridge.erl +++ b/apps/emqx_bridge/src/emqx_bridge.erl @@ -90,7 +90,7 @@ T == oracle; T == iotdb; T == kinesis_producer; - T == greptimedb_grpc_v1 + T == greptimedb ). -define(ROOT_KEY, bridges). diff --git a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl index c6765e859..c23ffb6df 100644 --- a/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl +++ b/apps/emqx_bridge/src/schema/emqx_bridge_enterprise.erl @@ -50,7 +50,7 @@ api_schemas(Method) -> api_ref(emqx_bridge_iotdb, <<"iotdb">>, Method), api_ref(emqx_bridge_rabbitmq, <<"rabbitmq">>, Method), api_ref(emqx_bridge_kinesis, <<"kinesis_producer">>, Method ++ "_producer"), - api_ref(emqx_bridge_greptimedb, <<"greptimedb_grpc_v1">>, Method ++ "_grpc_v1") + api_ref(emqx_bridge_greptimedb, <<"greptimedb">>, Method ++ "_grpc_v1") ]. schema_modules() -> @@ -124,7 +124,7 @@ resource_type(oracle) -> emqx_oracle; resource_type(iotdb) -> emqx_bridge_iotdb_impl; resource_type(rabbitmq) -> emqx_bridge_rabbitmq_connector; resource_type(kinesis_producer) -> emqx_bridge_kinesis_impl_producer; -resource_type(greptimedb_grpc_v1) -> emqx_bridge_greptimedb_connector. +resource_type(greptimedb) -> emqx_bridge_greptimedb_connector. fields(bridges) -> [ @@ -302,7 +302,7 @@ greptimedb_structs() -> } )} || Protocol <- [ - greptimedb_grpc_v1 + greptimedb ] ]. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl index 415544fcf..877e464dd 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.erl @@ -32,16 +32,16 @@ conn_bridge_examples(Method) -> [ #{ - <<"greptimedb_grpc_v1">> => #{ + <<"greptimedb">> => #{ summary => <<"Greptimedb HTTP API V2 Bridge">>, - value => values("greptimedb_grpc_v1", Method) + value => values("greptimedb", Method) } } ]. values(Protocol, get) -> values(Protocol, post); -values("greptimedb_grpc_v1", post) -> +values("greptimedb", post) -> SupportUint = <<"uint_value=${payload.uint_key}u,">>, TypeOpts = #{ bucket => <<"example_bucket">>, @@ -49,7 +49,7 @@ values("greptimedb_grpc_v1", post) -> token => <<"example_token">>, server => <<"127.0.0.1:4001">> }, - values(common, "greptimedb_grpc_v1", SupportUint, TypeOpts); + values(common, "greptimedb", SupportUint, TypeOpts); values(Protocol, put) -> values(Protocol, post). @@ -80,13 +80,13 @@ namespace() -> "bridge_greptimedb". roots() -> []. fields("post_grpc_v1") -> - method_fields(post, greptimedb_grpc_v1); + method_fields(post, greptimedb); fields("put_grpc_v1") -> - method_fields(put, greptimedb_grpc_v1); + method_fields(put, greptimedb); fields("get_grpc_v1") -> - method_fields(get, greptimedb_grpc_v1); + method_fields(get, greptimedb); fields(Type) when - Type == greptimedb_grpc_v1 + Type == greptimedb -> greptimedb_bridge_common_fields() ++ connector_fields(Type). @@ -125,8 +125,8 @@ desc("config") -> ?DESC("desc_config"); desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" -> ["Configuration for Greptimedb using `", string:to_upper(Method), "` method."]; -desc(greptimedb_grpc_v1) -> - ?DESC(emqx_bridge_greptimedb_connector, "greptimedb_grpc_v1"); +desc(greptimedb) -> + ?DESC(emqx_bridge_greptimedb_connector, "greptimedb"); desc(_) -> undefined. diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index a31606bbc..7473f9690 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -119,7 +119,7 @@ roots() -> {config, #{ type => hoconsc:union( [ - hoconsc:ref(?MODULE, greptimedb_grpc_v1) + hoconsc:ref(?MODULE, greptimedb) ] ) }} @@ -134,7 +134,7 @@ fields(common) -> required => false, default => ms, desc => ?DESC("precision") })} ]; -fields(greptimedb_grpc_v1) -> +fields(greptimedb) -> fields(common) ++ [ {dbname, mk(binary(), #{required => true, desc => ?DESC("dbname")})}, @@ -159,8 +159,8 @@ server() -> desc(common) -> ?DESC("common"); -desc(greptimedb_grpc_v1) -> - ?DESC("greptimedb_grpc_v1"). +desc(greptimedb) -> + ?DESC("greptimedb"). %% ------------------------------------------------------------------------------------------------- %% internal functions @@ -613,7 +613,7 @@ desc_test_() -> ), ?_assertMatch( {desc, _, _}, - desc(greptimedb_grpc_v1) + desc(greptimedb) ), ?_assertMatch( {desc, _, _}, diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index f563b7e71..25bdb6b76 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -191,7 +191,7 @@ greptimedb_config(grpcv1 = Type, GreptimedbHost, GreptimedbPort, Config) -> WriteSyntax = example_write_syntax(), ConfigString = io_lib:format( - "bridges.greptimedb_grpc_v1.~s {\n" + "bridges.greptimedb.~s {\n" " enable = true\n" " server = \"~p:~b\"\n" " dbname = public\n" @@ -229,7 +229,7 @@ parse_and_check(ConfigString, Type, Name) -> Config. greptimedb_type_bin(grpcv1) -> - <<"greptimedb_grpc_v1">>. + <<"greptimedb">>. create_bridge(Config) -> create_bridge(Config, _Overrides = #{}). @@ -487,7 +487,7 @@ t_start_ok_timestamp_write_syntax(Config) -> GreptimedbConfigString0 = ?config(greptimedb_config_string, Config), GreptimedbTypeCfg = case GreptimedbType of - grpcv1 -> "greptimedb_grpc_v1" + grpcv1 -> "greptimedb" end, WriteSyntax = %% N.B.: this single space characters are relevant @@ -521,7 +521,7 @@ t_start_ok_no_subject_tags_write_syntax(Config) -> GreptimedbConfigString0 = ?config(greptimedb_config_string, Config), GreptimedbTypeCfg = case GreptimedbType of - grpcv1 -> "greptimedb_grpc_v1" + grpcv1 -> "greptimedb" end, WriteSyntax = %% N.B.: this single space characters are relevant @@ -641,7 +641,7 @@ t_bad_timestamp(Config) -> GreptimedbConfigString0 = ?config(greptimedb_config_string, Config), GreptimedbTypeCfg = case GreptimedbType of - grpcv1 -> "greptimedb_grpc_v1" + grpcv1 -> "greptimedb" end, WriteSyntax = %% N.B.: this single space characters are relevant diff --git a/rel/i18n/emqx_bridge_greptimedb_connector.hocon b/rel/i18n/emqx_bridge_greptimedb_connector.hocon index 87370b211..9cb10951f 100644 --- a/rel/i18n/emqx_bridge_greptimedb_connector.hocon +++ b/rel/i18n/emqx_bridge_greptimedb_connector.hocon @@ -6,10 +6,10 @@ dbname.desc: dbname.label: """Database""" -greptimedb_grpc_v1.desc: +greptimedb.desc: """GreptimeDB's protocol. Support GreptimeDB v1.8 and before.""" -greptimedb_grpc_v1.label: +greptimedb.label: """HTTP API Protocol""" password.desc: From b34374c26ff8e7af866a3fc291d2993ee3c863f0 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 21 Jul 2023 14:59:24 +0800 Subject: [PATCH 16/21] chore: by CR comments --- .tool-versions | 2 +- .../src/emqx_bridge_greptimedb_connector.erl | 20 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.tool-versions b/.tool-versions index 0dbab2a1d..3a2251dc8 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 25.3.2.3 +erlang 25.3.2-1 elixir 1.14.5-otp-25 diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 7473f9690..c021e4354 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -32,7 +32,9 @@ ]). %% only for test +-ifdef(TEST). -export([is_unrecoverable_error/1]). +-endif. -type ts_precision() :: ns | us | ms | s. @@ -186,8 +188,8 @@ start_client(InstId, Config) -> msg => "start greptimedb connector error", connector => InstId, error => E, - reason => R, - stack => S + reason => emqx_utils:redact(R), + stack => emqx_utils:redact(S) }), {error, R} end. @@ -342,7 +344,7 @@ to_config(Lines, Precision) -> to_config([], Acc, _Precision) -> lists:reverse(Acc); to_config([Item0 | Rest], Acc, Precision) -> - Ts0 = maps:get(timestamp, Item0, undefined), + Ts0 = maps:get(timestamp, Item0, ?DEFAULT_TIMESTAMP_TMPL), {Ts, FromPrecision, ToPrecision} = preproc_tmpl_timestamp(Ts0, Precision), Item = #{ measurement => emqx_placeholder:preproc_tmpl(maps:get(measurement, Item0)), @@ -374,7 +376,11 @@ preproc_tmpl_timestamp(Ts, Precision) when is_binary(Ts) -> {emqx_placeholder:preproc_tmpl(Ts), Precision, Precision}. to_kv_config(KVfields) -> - maps:fold(fun to_maps_config/3, #{}, proplists:to_map(KVfields)). + lists:foldl( + fun({K, V}, Acc) -> to_maps_config(K, V, Acc) end, + #{}, + KVfields + ). to_maps_config(K, V, Res) -> NK = emqx_placeholder:preproc_tmpl(bin(K)), @@ -391,6 +397,8 @@ parse_batch_data(InstId, BatchData, SyntaxLines) -> {[Points | ListOfPoints], ErrAccIn}; {error, ErrorPoints} -> log_error_points(InstId, ErrorPoints), + {ListOfPoints, ErrAccIn + 1}; + _ -> {ListOfPoints, ErrAccIn + 1} end end, @@ -522,8 +530,6 @@ value_type([UInt, <<"u">>]) when is_integer(UInt) -> greptimedb_values:uint64_value(UInt); -value_type([Float]) when is_float(Float) -> - Float; value_type([<<"t">>]) -> greptimedb_values:boolean_value(true); value_type([<<"T">>]) -> @@ -544,6 +550,8 @@ value_type([<<"FALSE">>]) -> greptimedb_values:boolean_value(false); value_type([<<"False">>]) -> greptimedb_values:boolean_value(false); +value_type([Float]) when is_float(Float) -> + Float; value_type(Val) -> #{values => #{string_values => Val}, datatype => 'STRING'}. From ba9dcbcff0471c29cead4f40dc3e03edcafdccd3 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 21 Jul 2023 15:39:41 +0800 Subject: [PATCH 17/21] chore: style --- changes/ee/feat-10647.en.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changes/ee/feat-10647.en.md b/changes/ee/feat-10647.en.md index 2b341fa4b..b42ef1f94 100644 --- a/changes/ee/feat-10647.en.md +++ b/changes/ee/feat-10647.en.md @@ -1 +1 @@ -Add enterprise data bridge for [GreptimeDB](https://github.com/GreptimeTeam/greptimedb). \ No newline at end of file +Add enterprise data bridge for [GreptimeDB](https://github.com/GreptimeTeam/greptimedb). From 2ea903c5aca90f1bfc1166ec95f8305b0a4725aa Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 21 Jul 2023 17:12:19 +0800 Subject: [PATCH 18/21] fix: static checks failures --- .../src/emqx_bridge_greptimedb_connector.erl | 6 ++---- .../test/emqx_bridge_greptimedb_SUITE.erl | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index c021e4354..666073913 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -271,7 +271,7 @@ protocol_config( } = Config ) -> [ - {dbname, DbName} + {dbname, str(DbName)} ] ++ auth(Config) ++ ssl_config(SSL). @@ -288,7 +288,7 @@ ssl_config(SSL = #{enable := true}) -> auth(#{username := Username, password := Password}) -> [ - {auth, {basic, #{username => Username, password => Password}}} + {auth, {basic, #{username => str(Username), password => str(Password)}}} ]; auth(_) -> []. @@ -397,8 +397,6 @@ parse_batch_data(InstId, BatchData, SyntaxLines) -> {[Points | ListOfPoints], ErrAccIn}; {error, ErrorPoints} -> log_error_points(InstId, ErrorPoints), - {ListOfPoints, ErrAccIn + 1}; - _ -> {ListOfPoints, ErrAccIn + 1} end end, diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index 25bdb6b76..bcd57f530 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -49,7 +49,6 @@ end_per_suite(_Config) -> emqx_conf, emqx_bridge, emqx_resource, emqx_rule_engine ]), _ = application:stop(emqx_connector), - _ = application:stop(greptimedb), ok. init_per_group(GreptimedbType, Config0) when @@ -145,6 +144,7 @@ end_per_group(Group, Config) when emqx_common_test_helpers:reset_proxy(ProxyHost, ProxyPort), ehttpc_sup:stop_pool(EHttpcPoolName), delete_bridge(Config), + _ = application:stop(greptimedb), ok; end_per_group(_Group, _Config) -> ok. From cd9d5f287ee2de93e76a5ed33dcc2d3633637240 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Fri, 21 Jul 2023 19:16:58 +0800 Subject: [PATCH 19/21] chore: adds auto_reconnect for ecpool --- .../src/emqx_bridge_greptimedb_connector.erl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl index 666073913..4be100594 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb_connector.erl @@ -51,6 +51,8 @@ -define(DEFAULT_TIMESTAMP_TMPL, "${timestamp}"). +-define(AUTO_RECONNECT_S, 1). + %% ------------------------------------------------------------------------------------------------- %% resource callback callback_mode() -> always_sync. @@ -261,6 +263,7 @@ client_config( {pool_size, erlang:system_info(schedulers)}, {pool, InstId}, {pool_type, random}, + {auto_reconnect, ?AUTO_RECONNECT_S}, {timeunit, maps:get(precision, Config, ms)} ] ++ protocol_config(Config). From 9f200120c2f2b4955db0e587fdc6b57de43c6625 Mon Sep 17 00:00:00 2001 From: Dennis Zhuang Date: Mon, 24 Jul 2023 12:04:29 +0800 Subject: [PATCH 20/21] feat: use http port to detect server availability --- .../test/emqx_bridge_greptimedb_SUITE.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl index bcd57f530..d4bc5b01e 100644 --- a/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl +++ b/apps/emqx_bridge_greptimedb/test/emqx_bridge_greptimedb_SUITE.erl @@ -80,7 +80,7 @@ init_per_group(GreptimedbType, Config0) when proxy_name => "greptimedb_tls" } end, - case emqx_common_test_helpers:is_tcp_server_available(GreptimedbHost, GreptimedbPort) of + case emqx_common_test_helpers:is_tcp_server_available(GreptimedbHost, GreptimedbHttpPort) of true -> ProxyHost = os:getenv("PROXY_HOST", "toxiproxy"), ProxyPort = list_to_integer(os:getenv("PROXY_PORT", "8474")), @@ -93,7 +93,7 @@ init_per_group(GreptimedbType, Config0) when {Name, ConfigString, GreptimedbConfig} = greptimedb_config( grpcv1, GreptimedbHost, GreptimedbPort, Config ), - EHttpcPoolNameBin = <<(atom_to_binary(?MODULE))/binary, "_grpcv1">>, + EHttpcPoolNameBin = <<(atom_to_binary(?MODULE))/binary, "_http">>, EHttpcPoolName = binary_to_atom(EHttpcPoolNameBin), {EHttpcTransport, EHttpcTransportOpts} = case UseTLS of From 8439ce0e844655471f9501babbc31c69fb9261d2 Mon Sep 17 00:00:00 2001 From: firest Date: Mon, 24 Jul 2023 18:41:28 +0800 Subject: [PATCH 21/21] chore: update app version && reboot_lists --- apps/emqx/src/emqx.app.src | 2 +- apps/emqx_bridge/src/emqx_bridge.app.src | 2 +- apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src | 1 - apps/emqx_machine/priv/reboot_lists.eterm | 1 + apps/emqx_machine/src/emqx_machine.app.src | 2 +- 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/emqx/src/emqx.app.src b/apps/emqx/src/emqx.app.src index 47f1ae4b4..5ee4e2688 100644 --- a/apps/emqx/src/emqx.app.src +++ b/apps/emqx/src/emqx.app.src @@ -2,7 +2,7 @@ {application, emqx, [ {id, "emqx"}, {description, "EMQX Core"}, - {vsn, "5.1.2"}, + {vsn, "5.1.3"}, {modules, []}, {registered, []}, {applications, [ diff --git a/apps/emqx_bridge/src/emqx_bridge.app.src b/apps/emqx_bridge/src/emqx_bridge.app.src index 11d199c9d..fabf4d334 100644 --- a/apps/emqx_bridge/src/emqx_bridge.app.src +++ b/apps/emqx_bridge/src/emqx_bridge.app.src @@ -1,7 +1,7 @@ %% -*- mode: erlang -*- {application, emqx_bridge, [ {description, "EMQX bridges"}, - {vsn, "0.1.23"}, + {vsn, "0.1.24"}, {registered, [emqx_bridge_sup]}, {mod, {emqx_bridge_app, []}}, {applications, [ diff --git a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src index 53053c80b..c048a0d0c 100644 --- a/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src +++ b/apps/emqx_bridge_greptimedb/src/emqx_bridge_greptimedb.app.src @@ -6,7 +6,6 @@ kernel, stdlib, emqx_resource, - emqx_bridge, greptimedb ]}, {env, []}, diff --git a/apps/emqx_machine/priv/reboot_lists.eterm b/apps/emqx_machine/priv/reboot_lists.eterm index 500a47d8f..92f6b4bbd 100644 --- a/apps/emqx_machine/priv/reboot_lists.eterm +++ b/apps/emqx_machine/priv/reboot_lists.eterm @@ -85,6 +85,7 @@ emqx_bridge_opents, emqx_bridge_clickhouse, emqx_bridge_dynamo, + emqx_bridge_greptimedb, emqx_bridge_hstreamdb, emqx_bridge_influxdb, emqx_bridge_iotdb, diff --git a/apps/emqx_machine/src/emqx_machine.app.src b/apps/emqx_machine/src/emqx_machine.app.src index e81d4b53f..9a9dedc28 100644 --- a/apps/emqx_machine/src/emqx_machine.app.src +++ b/apps/emqx_machine/src/emqx_machine.app.src @@ -3,7 +3,7 @@ {id, "emqx_machine"}, {description, "The EMQX Machine"}, % strict semver, bump manually! - {vsn, "0.2.8"}, + {vsn, "0.2.9"}, {modules, []}, {registered, []}, {applications, [kernel, stdlib, emqx_ctl]},