diff --git a/Makefile b/Makefile index 26c04b277..557ecadfc 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ PROFILE ?= emqx PROFILES := emqx emqx-edge PKG_PROFILES := emqx-pkg emqx-edge-pkg -export REBAR_GIT_CLONE_OPTIONS=--depth=1 +export REBAR_GIT_CLONE_OPTIONS += --depth=1 .PHONY: default default: $(REBAR) $(PROFILE) diff --git a/apps/emqx_auth_http/rebar.config b/apps/emqx_auth_http/rebar.config index 7b1f06cca..705557fe5 100644 --- a/apps/emqx_auth_http/rebar.config +++ b/apps/emqx_auth_http/rebar.config @@ -1 +1,27 @@ -{deps, []}. \ No newline at end of file +{deps, []}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. + +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} + ]} + ]} + ]}. + diff --git a/apps/emqx_auth_jwt/rebar.config b/apps/emqx_auth_jwt/rebar.config index 0728a0b95..f711075ba 100644 --- a/apps/emqx_auth_jwt/rebar.config +++ b/apps/emqx_auth_jwt/rebar.config @@ -1,3 +1,24 @@ {deps, [{jwerl, {git, "https://github.com/emqx/jwerl.git", {branch, "1.1.1"}}} ]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, [{emqx_ct_helpers, {git, "http://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}]} + ]} + ]}. diff --git a/apps/emqx_auth_ldap/.ci/docker-compose.yml b/apps/emqx_auth_ldap/.ci/docker-compose.yml deleted file mode 100644 index bba9b711f..000000000 --- a/apps/emqx_auth_ldap/.ci/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: '3' - -services: - erlang: - image: erlang:22.1 - volumes: - - ../:/emqx_auth_ldap - networks: - - emqx_bridge - depends_on: - - ldap_server - tty: true - - ldap_server: - build: ./emqx-ldap - image: emqx-ldap:1.0 - restart: always - ports: - - 389:389 - - 636:636 - networks: - - emqx_bridge - -networks: - emqx_bridge: - driver: bridge diff --git a/apps/emqx_auth_ldap/.ci/emqx-ldap/Dockerfile b/apps/emqx_auth_ldap/.ci/emqx-ldap/Dockerfile deleted file mode 100644 index 0a01572c4..000000000 --- a/apps/emqx_auth_ldap/.ci/emqx-ldap/Dockerfile +++ /dev/null @@ -1,26 +0,0 @@ -FROM buildpack-deps:stretch - -ENV VERSION=2.4.50 - -RUN apt-get update && apt-get install -y groff groff-base -RUN wget ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-${VERSION}.tgz \ - && gunzip -c openldap-${VERSION}.tgz | tar xvfB - \ - && cd openldap-${VERSION} \ - && ./configure && make depend && make && make install \ - && cd .. && rm -rf openldap-${VERSION} - -COPY ./slapd.conf /usr/local/etc/openldap/slapd.conf -COPY ./emqx.io.ldif /usr/local/etc/openldap/schema/emqx.io.ldif -COPY ./emqx.schema /usr/local/etc/openldap/schema/emqx.schema -COPY ./*.pem /usr/local/etc/openldap/ - -RUN mkdir -p /usr/local/etc/openldap/data \ - && slapadd -l /usr/local/etc/openldap/schema/emqx.io.ldif -f /usr/local/etc/openldap/slapd.conf - -WORKDIR /usr/local/etc/openldap - -EXPOSE 389 636 - -ENTRYPOINT ["/usr/local/libexec/slapd", "-h", "ldap:/// ldaps:///", "-d", "3", "-f", "/usr/local/etc/openldap/slapd.conf"] - -CMD [] diff --git a/apps/emqx_auth_ldap/.ci/emqx-ldap/slapd.conf b/apps/emqx_auth_ldap/.ci/emqx-ldap/slapd.conf deleted file mode 100644 index d6ba20caa..000000000 --- a/apps/emqx_auth_ldap/.ci/emqx-ldap/slapd.conf +++ /dev/null @@ -1,16 +0,0 @@ -include /usr/local/etc/openldap/schema/core.schema -include /usr/local/etc/openldap/schema/cosine.schema -include /usr/local/etc/openldap/schema/inetorgperson.schema -include /usr/local/etc/openldap/schema/ppolicy.schema -include /usr/local/etc/openldap/schema/emqx.schema - -TLSCACertificateFile /usr/local/etc/openldap/cacert.pem -TLSCertificateFile /usr/local/etc/openldap/cert.pem -TLSCertificateKeyFile /usr/local/etc/openldap/key.pem - -database bdb -suffix "dc=emqx,dc=io" -rootdn "cn=root,dc=emqx,dc=io" -rootpw {SSHA}eoF7NhNrejVYYyGHqnt+MdKNBh4r1w3W - -directory /usr/local/etc/openldap/data diff --git a/apps/emqx_auth_ldap/rebar.config b/apps/emqx_auth_ldap/rebar.config index 8fe128ac5..48eaf812f 100644 --- a/apps/emqx_auth_ldap/rebar.config +++ b/apps/emqx_auth_ldap/rebar.config @@ -1,3 +1,25 @@ {deps, [{eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}} ]}. + +{profiles, + [{test, + [{deps, [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}]} + ]} + ]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + diff --git a/apps/emqx_auth_mongo/rebar.config b/apps/emqx_auth_mongo/rebar.config index f913dd272..4c9417195 100644 --- a/apps/emqx_auth_mongo/rebar.config +++ b/apps/emqx_auth_mongo/rebar.config @@ -1,2 +1,32 @@ {deps, - [{mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.7"}}}]}. \ No newline at end of file + [{mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.7"}}}]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + compressed, + {parse_transform} + ]}. +{overrides, [{add, [{erl_opts, [compressed]}]}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions + ]}. + +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helper", {tag, "1.2.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} + ]}, + {erl_opts, [debug_info]} + ]} + ]}. diff --git a/apps/emqx_auth_mysql/rebar.config b/apps/emqx_auth_mysql/rebar.config index bc21ad1b6..18841d104 100644 --- a/apps/emqx_auth_mysql/rebar.config +++ b/apps/emqx_auth_mysql/rebar.config @@ -2,3 +2,33 @@ [ {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.6.1"}}} ]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + compressed, + {parse_transform} + ]}. +{overrides, [{add, [{erl_opts, [compressed]}]}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions + ]}. + +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} + ]}, + {erl_opts, [debug_info]} + ]} + ]}. diff --git a/apps/emqx_auth_pgsql/.ci/docker-compose.yml b/apps/emqx_auth_pgsql/.ci/docker-compose.yml deleted file mode 100644 index 8782a841d..000000000 --- a/apps/emqx_auth_pgsql/.ci/docker-compose.yml +++ /dev/null @@ -1,30 +0,0 @@ -version: '3' - -services: - erlang: - image: erlang:22.3 - volumes: - - ../:/emqx_auth_pgsql - networks: - - emqx_bridge - depends_on: - - pgsql_server - tty: true - - pgsql_server: - build: - context: ./pgsql - args: - BUILD_FROM: postgres:${PGSQL_TAG} - image: emqx-pgsql - restart: always - environment: - POSTGRES_PASSWORD: public - POSTGRES_USER: root - POSTGRES_DB: mqtt - networks: - - emqx_bridge - -networks: - emqx_bridge: - driver: bridge diff --git a/apps/emqx_auth_pgsql/.ci/pgsql/Dockerfile b/apps/emqx_auth_pgsql/.ci/pgsql/Dockerfile deleted file mode 100644 index 785bb875f..000000000 --- a/apps/emqx_auth_pgsql/.ci/pgsql/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -ARG BUILD_FROM=postgres:11 -FROM ${BUILD_FROM} -COPY pg.conf /etc/postgresql/postgresql.conf -COPY server-cert.pem /etc/postgresql/server-cert.pem -COPY server-key.pem /etc/postgresql/server-key.pem -RUN chown -R postgres:postgres /etc/postgresql \ - && chmod 600 /etc/postgresql/*.pem -CMD ["-c", "config_file=/etc/postgresql/postgresql.conf"] diff --git a/apps/emqx_auth_pgsql/rebar.config b/apps/emqx_auth_pgsql/rebar.config index a56845649..20f84fa83 100644 --- a/apps/emqx_auth_pgsql/rebar.config +++ b/apps/emqx_auth_pgsql/rebar.config @@ -1,3 +1,31 @@ {deps, [{epgsql, {git, "https://github.com/epgsql/epgsql", {tag, "4.4.0"}}} - ]}. \ No newline at end of file + ]}. + +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + compressed + ]}. +{overrides, [{add, [{erl_opts, [compressed]}]}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions + ]}. + +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helper", {branch, "1.2.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} + ]}, + {erl_opts, [debug_info]} + ]} + ]}. diff --git a/apps/emqx_auth_redis/.ci/docker-compose.yml b/apps/emqx_auth_redis/.ci/docker-compose.yml deleted file mode 100644 index 27794546e..000000000 --- a/apps/emqx_auth_redis/.ci/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: '3' - -services: - erlang: - image: erlang:22.1 - volumes: - - ../:/emqx_auth_redis - networks: - - emqx_bridge - depends_on: - - redis_server - tty: true - - redis_server: - build: - context: ./emqx-redis - args: - BUILD_FROM: redis:${REDIS_TAG} - image: emqx-redis - restart: always - networks: - - emqx_bridge - -networks: - emqx_bridge: - driver: bridge diff --git a/apps/emqx_auth_redis/.ci/emqx-redis/Dockerfile b/apps/emqx_auth_redis/.ci/emqx-redis/Dockerfile deleted file mode 100644 index 9fe51e86e..000000000 --- a/apps/emqx_auth_redis/.ci/emqx-redis/Dockerfile +++ /dev/null @@ -1,4 +0,0 @@ -ARG BUILD_FROM=redis:5 -FROM ${BUILD_FROM} -COPY redis.conf /usr/local/etc/redis/redis.conf -CMD [ "redis-server", "/usr/local/etc/redis/redis.conf" ] diff --git a/apps/emqx_auth_redis/.ci/emqx-redis/redis.conf b/apps/emqx_auth_redis/.ci/emqx-redis/redis.conf deleted file mode 100644 index 8415f9d5f..000000000 --- a/apps/emqx_auth_redis/.ci/emqx-redis/redis.conf +++ /dev/null @@ -1,1377 +0,0 @@ -# Redis configuration file example. -# -# Note that in order to read the configuration file, Redis must be -# started with the file path as first argument: -# -# ./redis-server /path/to/redis.conf - -# Note on units: when memory size is needed, it is possible to specify -# it in the usual form of 1k 5GB 4M and so forth: -# -# 1k => 1000 bytes -# 1kb => 1024 bytes -# 1m => 1000000 bytes -# 1mb => 1024*1024 bytes -# 1g => 1000000000 bytes -# 1gb => 1024*1024*1024 bytes -# -# units are case insensitive so 1GB 1Gb 1gB are all the same. - -################################## INCLUDES ################################### - -# Include one or more other config files here. This is useful if you -# have a standard template that goes to all Redis servers but also need -# to customize a few per-server settings. Include files can include -# other files, so use this wisely. -# -# Notice option "include" won't be rewritten by command "CONFIG REWRITE" -# from admin or Redis Sentinel. Since Redis always uses the last processed -# line as value of a configuration directive, you'd better put includes -# at the beginning of this file to avoid overwriting config change at runtime. -# -# If instead you are interested in using includes to override configuration -# options, it is better to use include as the last line. -# -# include /path/to/local.conf -# include /path/to/other.conf - -################################## MODULES ##################################### - -# Load modules at startup. If the server is not able to load modules -# it will abort. It is possible to use multiple loadmodule directives. -# -# loadmodule /path/to/my_module.so -# loadmodule /path/to/other_module.so - -################################## NETWORK ##################################### - -# By default, if no "bind" configuration directive is specified, Redis listens -# for connections from all the network interfaces available on the server. -# It is possible to listen to just one or multiple selected interfaces using -# the "bind" configuration directive, followed by one or more IP addresses. -# -# Examples: -# -# bind 192.168.1.100 10.0.0.1 -# bind 127.0.0.1 ::1 -# -# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the -# internet, binding to all the interfaces is dangerous and will expose the -# instance to everybody on the internet. So by default we uncomment the -# following bind directive, that will force Redis to listen only into -# the IPv4 loopback interface address (this means Redis will be able to -# accept connections only from clients running into the same computer it -# is running). -# -# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES -# JUST COMMENT THE FOLLOWING LINE. -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -bind 0.0.0.0 :: - -# Protected mode is a layer of security protection, in order to avoid that -# Redis instances left open on the internet are accessed and exploited. -# -# When protected mode is on and if: -# -# 1) The server is not binding explicitly to a set of addresses using the -# "bind" directive. -# 2) No password is configured. -# -# The server only accepts connections from clients connecting from the -# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain -# sockets. -# -# By default protected mode is enabled. You should disable it only if -# you are sure you want clients from other hosts to connect to Redis -# even if no authentication is configured, nor a specific set of interfaces -# are explicitly listed using the "bind" directive. -protected-mode no - -# Accept connections on the specified port, default is 6379 (IANA #815344). -# If port 0 is specified Redis will not listen on a TCP socket. -port 6379 - -# TCP listen() backlog. -# -# In high requests-per-second environments you need an high backlog in order -# to avoid slow clients connections issues. Note that the Linux kernel -# will silently truncate it to the value of /proc/sys/net/core/somaxconn so -# make sure to raise both the value of somaxconn and tcp_max_syn_backlog -# in order to get the desired effect. -tcp-backlog 511 - -# Unix socket. -# -# Specify the path for the Unix socket that will be used to listen for -# incoming connections. There is no default, so Redis will not listen -# on a unix socket when not specified. -# -# unixsocket /tmp/redis.sock -# unixsocketperm 700 - -# Close the connection after a client is idle for N seconds (0 to disable) -timeout 0 - -# TCP keepalive. -# -# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence -# of communication. This is useful for two reasons: -# -# 1) Detect dead peers. -# 2) Take the connection alive from the point of view of network -# equipment in the middle. -# -# On Linux, the specified value (in seconds) is the period used to send ACKs. -# Note that to close the connection the double of the time is needed. -# On other kernels the period depends on the kernel configuration. -# -# A reasonable value for this option is 300 seconds, which is the new -# Redis default starting with Redis 3.2.1. -tcp-keepalive 300 - -################################# GENERAL ##################################### - -# By default Redis does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. -daemonize no - -# If you run Redis from upstart or systemd, Redis can interact with your -# supervision tree. Options: -# supervised no - no supervision interaction -# supervised upstart - signal upstart by putting Redis into SIGSTOP mode -# supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET -# supervised auto - detect upstart or systemd method based on -# UPSTART_JOB or NOTIFY_SOCKET environment variables -# Note: these supervision methods only signal "process is ready." -# They do not enable continuous liveness pings back to your supervisor. -supervised no - -# If a pid file is specified, Redis writes it where specified at startup -# and removes it at exit. -# -# When the server runs non daemonized, no pid file is created if none is -# specified in the configuration. When the server is daemonized, the pid file -# is used even if not specified, defaulting to "/var/run/redis.pid". -# -# Creating a pid file is best effort: if Redis is not able to create it -# nothing bad happens, the server will start and run normally. -pidfile /var/run/redis_6379.pid - -# Specify the server verbosity level. -# This can be one of: -# debug (a lot of information, useful for development/testing) -# verbose (many rarely useful info, but not a mess like the debug level) -# notice (moderately verbose, what you want in production probably) -# warning (only very important / critical messages are logged) -loglevel notice - -# Specify the log file name. Also the empty string can be used to force -# Redis to log on the standard output. Note that if you use standard -# output for logging but daemonize, logs will be sent to /dev/null -logfile "" - -# To enable logging to the system logger, just set 'syslog-enabled' to yes, -# and optionally update the other syslog parameters to suit your needs. -# syslog-enabled no - -# Specify the syslog identity. -# syslog-ident redis - -# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. -# syslog-facility local0 - -# Set the number of databases. The default database is DB 0, you can select -# a different one on a per-connection basis using SELECT where -# dbid is a number between 0 and 'databases'-1 -databases 16 - -# By default Redis shows an ASCII art logo only when started to log to the -# standard output and if the standard output is a TTY. Basically this means -# that normally a logo is displayed only in interactive sessions. -# -# However it is possible to force the pre-4.0 behavior and always show a -# ASCII art logo in startup logs by setting the following option to yes. -always-show-logo yes - -################################ SNAPSHOTTING ################################ -# -# Save the DB on disk: -# -# save -# -# Will save the DB if both the given number of seconds and the given -# number of write operations against the DB occurred. -# -# In the example below the behaviour will be to save: -# after 900 sec (15 min) if at least 1 key changed -# after 300 sec (5 min) if at least 10 keys changed -# after 60 sec if at least 10000 keys changed -# -# Note: you can disable saving completely by commenting out all "save" lines. -# -# It is also possible to remove all the previously configured save -# points by adding a save directive with a single empty string argument -# like in the following example: -# -# save "" - -save 900 1 -save 300 10 -save 60 10000 - -# By default Redis will stop accepting writes if RDB snapshots are enabled -# (at least one save point) and the latest background save failed. -# This will make the user aware (in a hard way) that data is not persisting -# on disk properly, otherwise chances are that no one will notice and some -# disaster will happen. -# -# If the background saving process will start working again Redis will -# automatically allow writes again. -# -# However if you have setup your proper monitoring of the Redis server -# and persistence, you may want to disable this feature so that Redis will -# continue to work as usual even if there are problems with disk, -# permissions, and so forth. -stop-writes-on-bgsave-error yes - -# Compress string objects using LZF when dump .rdb databases? -# For default that's set to 'yes' as it's almost always a win. -# If you want to save some CPU in the saving child set it to 'no' but -# the dataset will likely be bigger if you have compressible values or keys. -rdbcompression yes - -# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. -# This makes the format more resistant to corruption but there is a performance -# hit to pay (around 10%) when saving and loading RDB files, so you can disable it -# for maximum performances. -# -# RDB files created with checksum disabled have a checksum of zero that will -# tell the loading code to skip the check. -rdbchecksum yes - -# The filename where to dump the DB -dbfilename dump.rdb - -# The working directory. -# -# The DB will be written inside this directory, with the filename specified -# above using the 'dbfilename' configuration directive. -# -# The Append Only File will also be created inside this directory. -# -# Note that you must specify a directory here, not a file name. -dir ./ - -################################# REPLICATION ################################# - -# Master-Replica replication. Use replicaof to make a Redis instance a copy of -# another Redis server. A few things to understand ASAP about Redis replication. -# -# +------------------+ +---------------+ -# | Master | ---> | Replica | -# | (receive writes) | | (exact copy) | -# +------------------+ +---------------+ -# -# 1) Redis replication is asynchronous, but you can configure a master to -# stop accepting writes if it appears to be not connected with at least -# a given number of replicas. -# 2) Redis replicas are able to perform a partial resynchronization with the -# master if the replication link is lost for a relatively small amount of -# time. You may want to configure the replication backlog size (see the next -# sections of this file) with a sensible value depending on your needs. -# 3) Replication is automatic and does not need user intervention. After a -# network partition replicas automatically try to reconnect to masters -# and resynchronize with them. -# -# replicaof - -# If the master is password protected (using the "requirepass" configuration -# directive below) it is possible to tell the replica to authenticate before -# starting the replication synchronization process, otherwise the master will -# refuse the replica request. -# -# masterauth - -# When a replica loses its connection with the master, or when the replication -# is still in progress, the replica can act in two different ways: -# -# 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will -# still reply to client requests, possibly with out of date data, or the -# data set may just be empty if this is the first synchronization. -# -# 2) if replica-serve-stale-data is set to 'no' the replica will reply with -# an error "SYNC with master in progress" to all the kind of commands -# but to INFO, replicaOF, AUTH, PING, SHUTDOWN, REPLCONF, ROLE, CONFIG, -# SUBSCRIBE, UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, -# COMMAND, POST, HOST: and LATENCY. -# -replica-serve-stale-data yes - -# You can configure a replica instance to accept writes or not. Writing against -# a replica instance may be useful to store some ephemeral data (because data -# written on a replica will be easily deleted after resync with the master) but -# may also cause problems if clients are writing to it because of a -# misconfiguration. -# -# Since Redis 2.6 by default replicas are read-only. -# -# Note: read only replicas are not designed to be exposed to untrusted clients -# on the internet. It's just a protection layer against misuse of the instance. -# Still a read only replica exports by default all the administrative commands -# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve -# security of read only replicas using 'rename-command' to shadow all the -# administrative / dangerous commands. -replica-read-only yes - -# Replication SYNC strategy: disk or socket. -# -# ------------------------------------------------------- -# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY -# ------------------------------------------------------- -# -# New replicas and reconnecting replicas that are not able to continue the replication -# process just receiving differences, need to do what is called a "full -# synchronization". An RDB file is transmitted from the master to the replicas. -# The transmission can happen in two different ways: -# -# 1) Disk-backed: The Redis master creates a new process that writes the RDB -# file on disk. Later the file is transferred by the parent -# process to the replicas incrementally. -# 2) Diskless: The Redis master creates a new process that directly writes the -# RDB file to replica sockets, without touching the disk at all. -# -# With disk-backed replication, while the RDB file is generated, more replicas -# can be queued and served with the RDB file as soon as the current child producing -# the RDB file finishes its work. With diskless replication instead once -# the transfer starts, new replicas arriving will be queued and a new transfer -# will start when the current one terminates. -# -# When diskless replication is used, the master waits a configurable amount of -# time (in seconds) before starting the transfer in the hope that multiple replicas -# will arrive and the transfer can be parallelized. -# -# With slow disks and fast (large bandwidth) networks, diskless replication -# works better. -repl-diskless-sync no - -# When diskless replication is enabled, it is possible to configure the delay -# the server waits in order to spawn the child that transfers the RDB via socket -# to the replicas. -# -# This is important since once the transfer starts, it is not possible to serve -# new replicas arriving, that will be queued for the next RDB transfer, so the server -# waits a delay in order to let more replicas arrive. -# -# The delay is specified in seconds, and by default is 5 seconds. To disable -# it entirely just set it to 0 seconds and the transfer will start ASAP. -repl-diskless-sync-delay 5 - -# Replicas send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_replica_period option. The default value is 10 -# seconds. -# -# repl-ping-replica-period 10 - -# The following option sets the replication timeout for: -# -# 1) Bulk transfer I/O during SYNC, from the point of view of replica. -# 2) Master timeout from the point of view of replicas (data, pings). -# 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). -# -# It is important to make sure that this value is greater than the value -# specified for repl-ping-replica-period otherwise a timeout will be detected -# every time there is low traffic between the master and the replica. -# -# repl-timeout 60 - -# Disable TCP_NODELAY on the replica socket after SYNC? -# -# If you select "yes" Redis will use a smaller number of TCP packets and -# less bandwidth to send data to replicas. But this can add a delay for -# the data to appear on the replica side, up to 40 milliseconds with -# Linux kernels using a default configuration. -# -# If you select "no" the delay for data to appear on the replica side will -# be reduced but more bandwidth will be used for replication. -# -# By default we optimize for low latency, but in very high traffic conditions -# or when the master and replicas are many hops away, turning this to "yes" may -# be a good idea. -repl-disable-tcp-nodelay no - -# Set the replication backlog size. The backlog is a buffer that accumulates -# replica data when replicas are disconnected for some time, so that when a replica -# wants to reconnect again, often a full resync is not needed, but a partial -# resync is enough, just passing the portion of data the replica missed while -# disconnected. -# -# The bigger the replication backlog, the longer the time the replica can be -# disconnected and later be able to perform a partial resynchronization. -# -# The backlog is only allocated once there is at least a replica connected. -# -# repl-backlog-size 1mb - -# After a master has no longer connected replicas for some time, the backlog -# will be freed. The following option configures the amount of seconds that -# need to elapse, starting from the time the last replica disconnected, for -# the backlog buffer to be freed. -# -# Note that replicas never free the backlog for timeout, since they may be -# promoted to masters later, and should be able to correctly "partially -# resynchronize" with the replicas: hence they should always accumulate backlog. -# -# A value of 0 means to never release the backlog. -# -# repl-backlog-ttl 3600 - -# The replica priority is an integer number published by Redis in the INFO output. -# It is used by Redis Sentinel in order to select a replica to promote into a -# master if the master is no longer working correctly. -# -# A replica with a low priority number is considered better for promotion, so -# for instance if there are three replicas with priority 10, 100, 25 Sentinel will -# pick the one with priority 10, that is the lowest. -# -# However a special priority of 0 marks the replica as not able to perform the -# role of master, so a replica with priority of 0 will never be selected by -# Redis Sentinel for promotion. -# -# By default the priority is 100. -replica-priority 100 - -# It is possible for a master to stop accepting writes if there are less than -# N replicas connected, having a lag less or equal than M seconds. -# -# The N replicas need to be in "online" state. -# -# The lag in seconds, that must be <= the specified value, is calculated from -# the last ping received from the replica, that is usually sent every second. -# -# This option does not GUARANTEE that N replicas will accept the write, but -# will limit the window of exposure for lost writes in case not enough replicas -# are available, to the specified number of seconds. -# -# For example to require at least 3 replicas with a lag <= 10 seconds use: -# -# min-replicas-to-write 3 -# min-replicas-max-lag 10 -# -# Setting one or the other to 0 disables the feature. -# -# By default min-replicas-to-write is set to 0 (feature disabled) and -# min-replicas-max-lag is set to 10. - -# A Redis master is able to list the address and port of the attached -# replicas in different ways. For example the "INFO replication" section -# offers this information, which is used, among other tools, by -# Redis Sentinel in order to discover replica instances. -# Another place where this info is available is in the output of the -# "ROLE" command of a master. -# -# The listed IP and address normally reported by a replica is obtained -# in the following way: -# -# IP: The address is auto detected by checking the peer address -# of the socket used by the replica to connect with the master. -# -# Port: The port is communicated by the replica during the replication -# handshake, and is normally the port that the replica is using to -# listen for connections. -# -# However when port forwarding or Network Address Translation (NAT) is -# used, the replica may be actually reachable via different IP and port -# pairs. The following two options can be used by a replica in order to -# report to its master a specific set of IP and port, so that both INFO -# and ROLE will report those values. -# -# There is no need to use both the options if you need to override just -# the port or the IP address. -# -# replica-announce-ip 5.5.5.5 -# replica-announce-port 1234 - -################################## SECURITY ################################### - -# Require clients to issue AUTH before processing any other -# commands. This might be useful in environments in which you do not trust -# others with access to the host running redis-server. -# -# This should stay commented out for backward compatibility and because most -# people do not need auth (e.g. they run their own servers). -# -# Warning: since Redis is pretty fast an outside user can try up to -# 150k passwords per second against a good box. This means that you should -# use a very strong password otherwise it will be very easy to break. -# -# requirepass foobared - -# Command renaming. -# -# It is possible to change the name of dangerous commands in a shared -# environment. For instance the CONFIG command may be renamed into something -# hard to guess so that it will still be available for internal-use tools -# but not available for general clients. -# -# Example: -# -# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 -# -# It is also possible to completely kill a command by renaming it into -# an empty string: -# -# rename-command CONFIG "" -# -# Please note that changing the name of commands that are logged into the -# AOF file or transmitted to replicas may cause problems. - -################################### CLIENTS #################################### - -# Set the max number of connected clients at the same time. By default -# this limit is set to 10000 clients, however if the Redis server is not -# able to configure the process file limit to allow for the specified limit -# the max number of allowed clients is set to the current file limit -# minus 32 (as Redis reserves a few file descriptors for internal uses). -# -# Once the limit is reached Redis will close all the new connections sending -# an error 'max number of clients reached'. -# -# maxclients 10000 - -############################## MEMORY MANAGEMENT ################################ - -# Set a memory usage limit to the specified amount of bytes. -# When the memory limit is reached Redis will try to remove keys -# according to the eviction policy selected (see maxmemory-policy). -# -# If Redis can't remove keys according to the policy, or if the policy is -# set to 'noeviction', Redis will start to reply with errors to commands -# that would use more memory, like SET, LPUSH, and so on, and will continue -# to reply to read-only commands like GET. -# -# This option is usually useful when using Redis as an LRU or LFU cache, or to -# set a hard memory limit for an instance (using the 'noeviction' policy). -# -# WARNING: If you have replicas attached to an instance with maxmemory on, -# the size of the output buffers needed to feed the replicas are subtracted -# from the used memory count, so that network problems / resyncs will -# not trigger a loop where keys are evicted, and in turn the output -# buffer of replicas is full with DELs of keys evicted triggering the deletion -# of more keys, and so forth until the database is completely emptied. -# -# In short... if you have replicas attached it is suggested that you set a lower -# limit for maxmemory so that there is some free RAM on the system for replica -# output buffers (but this is not needed if the policy is 'noeviction'). -# -# maxmemory - -# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory -# is reached. You can select among five behaviors: -# -# volatile-lru -> Evict using approximated LRU among the keys with an expire set. -# allkeys-lru -> Evict any key using approximated LRU. -# volatile-lfu -> Evict using approximated LFU among the keys with an expire set. -# allkeys-lfu -> Evict any key using approximated LFU. -# volatile-random -> Remove a random key among the ones with an expire set. -# allkeys-random -> Remove a random key, any key. -# volatile-ttl -> Remove the key with the nearest expire time (minor TTL) -# noeviction -> Don't evict anything, just return an error on write operations. -# -# LRU means Least Recently Used -# LFU means Least Frequently Used -# -# Both LRU, LFU and volatile-ttl are implemented using approximated -# randomized algorithms. -# -# Note: with any of the above policies, Redis will return an error on write -# operations, when there are no suitable keys for eviction. -# -# At the date of writing these commands are: set setnx setex append -# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd -# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby -# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby -# getset mset msetnx exec sort -# -# The default is: -# -# maxmemory-policy noeviction - -# LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated -# algorithms (in order to save memory), so you can tune it for speed or -# accuracy. For default Redis will check five keys and pick the one that was -# used less recently, you can change the sample size using the following -# configuration directive. -# -# The default of 5 produces good enough results. 10 Approximates very closely -# true LRU but costs more CPU. 3 is faster but not very accurate. -# -# maxmemory-samples 5 - -# Starting from Redis 5, by default a replica will ignore its maxmemory setting -# (unless it is promoted to master after a failover or manually). It means -# that the eviction of keys will be just handled by the master, sending the -# DEL commands to the replica as keys evict in the master side. -# -# This behavior ensures that masters and replicas stay consistent, and is usually -# what you want, however if your replica is writable, or you want the replica to have -# a different memory setting, and you are sure all the writes performed to the -# replica are idempotent, then you may change this default (but be sure to understand -# what you are doing). -# -# Note that since the replica by default does not evict, it may end using more -# memory than the one set via maxmemory (there are certain buffers that may -# be larger on the replica, or data structures may sometimes take more memory and so -# forth). So make sure you monitor your replicas and make sure they have enough -# memory to never hit a real out-of-memory condition before the master hits -# the configured maxmemory setting. -# -# replica-ignore-maxmemory yes - -############################# LAZY FREEING #################################### - -# Redis has two primitives to delete keys. One is called DEL and is a blocking -# deletion of the object. It means that the server stops processing new commands -# in order to reclaim all the memory associated with an object in a synchronous -# way. If the key deleted is associated with a small object, the time needed -# in order to execute the DEL command is very small and comparable to most other -# O(1) or O(log_N) commands in Redis. However if the key is associated with an -# aggregated value containing millions of elements, the server can block for -# a long time (even seconds) in order to complete the operation. -# -# For the above reasons Redis also offers non blocking deletion primitives -# such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and -# FLUSHDB commands, in order to reclaim memory in background. Those commands -# are executed in constant time. Another thread will incrementally free the -# object in the background as fast as possible. -# -# DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. -# It's up to the design of the application to understand when it is a good -# idea to use one or the other. However the Redis server sometimes has to -# delete keys or flush the whole database as a side effect of other operations. -# Specifically Redis deletes objects independently of a user call in the -# following scenarios: -# -# 1) On eviction, because of the maxmemory and maxmemory policy configurations, -# in order to make room for new data, without going over the specified -# memory limit. -# 2) Because of expire: when a key with an associated time to live (see the -# EXPIRE command) must be deleted from memory. -# 3) Because of a side effect of a command that stores data on a key that may -# already exist. For example the RENAME command may delete the old key -# content when it is replaced with another one. Similarly SUNIONSTORE -# or SORT with STORE option may delete existing keys. The SET command -# itself removes any old content of the specified key in order to replace -# it with the specified string. -# 4) During replication, when a replica performs a full resynchronization with -# its master, the content of the whole database is removed in order to -# load the RDB file just transferred. -# -# In all the above cases the default is to delete objects in a blocking way, -# like if DEL was called. However you can configure each case specifically -# in order to instead release memory in a non-blocking way like if UNLINK -# was called, using the following configuration directives: - -lazyfree-lazy-eviction no -lazyfree-lazy-expire no -lazyfree-lazy-server-del no -replica-lazy-flush no - -############################## APPEND ONLY MODE ############################### - -# By default Redis asynchronously dumps the dataset on disk. This mode is -# good enough in many applications, but an issue with the Redis process or -# a power outage may result into a few minutes of writes lost (depending on -# the configured save points). -# -# The Append Only File is an alternative persistence mode that provides -# much better durability. For instance using the default data fsync policy -# (see later in the config file) Redis can lose just one second of writes in a -# dramatic event like a server power outage, or a single write if something -# wrong with the Redis process itself happens, but the operating system is -# still running correctly. -# -# AOF and RDB persistence can be enabled at the same time without problems. -# If the AOF is enabled on startup Redis will load the AOF, that is the file -# with the better durability guarantees. -# -# Please check http://redis.io/topics/persistence for more information. - -appendonly no - -# The name of the append only file (default: "appendonly.aof") - -appendfilename "appendonly.aof" - -# The fsync() call tells the Operating System to actually write data on disk -# instead of waiting for more data in the output buffer. Some OS will really flush -# data on disk, some other OS will just try to do it ASAP. -# -# Redis supports three different modes: -# -# no: don't fsync, just let the OS flush the data when it wants. Faster. -# always: fsync after every write to the append only log. Slow, Safest. -# everysec: fsync only one time every second. Compromise. -# -# The default is "everysec", as that's usually the right compromise between -# speed and data safety. It's up to you to understand if you can relax this to -# "no" that will let the operating system flush the output buffer when -# it wants, for better performances (but if you can live with the idea of -# some data loss consider the default persistence mode that's snapshotting), -# or on the contrary, use "always" that's very slow but a bit safer than -# everysec. -# -# More details please check the following article: -# http://antirez.com/post/redis-persistence-demystified.html -# -# If unsure, use "everysec". - -# appendfsync always -appendfsync everysec -# appendfsync no - -# When the AOF fsync policy is set to always or everysec, and a background -# saving process (a background save or AOF log background rewriting) is -# performing a lot of I/O against the disk, in some Linux configurations -# Redis may block too long on the fsync() call. Note that there is no fix for -# this currently, as even performing fsync in a different thread will block -# our synchronous write(2) call. -# -# In order to mitigate this problem it's possible to use the following option -# that will prevent fsync() from being called in the main process while a -# BGSAVE or BGREWRITEAOF is in progress. -# -# This means that while another child is saving, the durability of Redis is -# the same as "appendfsync none". In practical terms, this means that it is -# possible to lose up to 30 seconds of log in the worst scenario (with the -# default Linux settings). -# -# If you have latency problems turn this to "yes". Otherwise leave it as -# "no" that is the safest pick from the point of view of durability. - -no-appendfsync-on-rewrite no - -# Automatic rewrite of the append only file. -# Redis is able to automatically rewrite the log file implicitly calling -# BGREWRITEAOF when the AOF log size grows by the specified percentage. -# -# This is how it works: Redis remembers the size of the AOF file after the -# latest rewrite (if no rewrite has happened since the restart, the size of -# the AOF at startup is used). -# -# This base size is compared to the current size. If the current size is -# bigger than the specified percentage, the rewrite is triggered. Also -# you need to specify a minimal size for the AOF file to be rewritten, this -# is useful to avoid rewriting the AOF file even if the percentage increase -# is reached but it is still pretty small. -# -# Specify a percentage of zero in order to disable the automatic AOF -# rewrite feature. - -auto-aof-rewrite-percentage 100 -auto-aof-rewrite-min-size 64mb - -# An AOF file may be found to be truncated at the end during the Redis -# startup process, when the AOF data gets loaded back into memory. -# This may happen when the system where Redis is running -# crashes, especially when an ext4 filesystem is mounted without the -# data=ordered option (however this can't happen when Redis itself -# crashes or aborts but the operating system still works correctly). -# -# Redis can either exit with an error when this happens, or load as much -# data as possible (the default now) and start if the AOF file is found -# to be truncated at the end. The following option controls this behavior. -# -# If aof-load-truncated is set to yes, a truncated AOF file is loaded and -# the Redis server starts emitting a log to inform the user of the event. -# Otherwise if the option is set to no, the server aborts with an error -# and refuses to start. When the option is set to no, the user requires -# to fix the AOF file using the "redis-check-aof" utility before to restart -# the server. -# -# Note that if the AOF file will be found to be corrupted in the middle -# the server will still exit with an error. This option only applies when -# Redis will try to read more data from the AOF file but not enough bytes -# will be found. -aof-load-truncated yes - -# When rewriting the AOF file, Redis is able to use an RDB preamble in the -# AOF file for faster rewrites and recoveries. When this option is turned -# on the rewritten AOF file is composed of two different stanzas: -# -# [RDB file][AOF tail] -# -# When loading Redis recognizes that the AOF file starts with the "REDIS" -# string and loads the prefixed RDB file, and continues loading the AOF -# tail. -aof-use-rdb-preamble yes - -################################ LUA SCRIPTING ############################### - -# Max execution time of a Lua script in milliseconds. -# -# If the maximum execution time is reached Redis will log that a script is -# still in execution after the maximum allowed time and will start to -# reply to queries with an error. -# -# When a long running script exceeds the maximum execution time only the -# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be -# used to stop a script that did not yet called write commands. The second -# is the only way to shut down the server in the case a write command was -# already issued by the script but the user doesn't want to wait for the natural -# termination of the script. -# -# Set it to 0 or a negative value for unlimited execution without warnings. -lua-time-limit 5000 - -################################ REDIS CLUSTER ############################### -# -# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -# WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however -# in order to mark it as "mature" we need to wait for a non trivial percentage -# of users to deploy it in production. -# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -# -# Normal Redis instances can't be part of a Redis Cluster; only nodes that are -# started as cluster nodes can. In order to start a Redis instance as a -# cluster node enable the cluster support uncommenting the following: -# -# cluster-enabled yes - -# Every cluster node has a cluster configuration file. This file is not -# intended to be edited by hand. It is created and updated by Redis nodes. -# Every Redis Cluster node requires a different cluster configuration file. -# Make sure that instances running in the same system do not have -# overlapping cluster configuration file names. -# -# cluster-config-file nodes-6379.conf - -# Cluster node timeout is the amount of milliseconds a node must be unreachable -# for it to be considered in failure state. -# Most other internal time limits are multiple of the node timeout. -# -# cluster-node-timeout 15000 - -# A replica of a failing master will avoid to start a failover if its data -# looks too old. -# -# There is no simple way for a replica to actually have an exact measure of -# its "data age", so the following two checks are performed: -# -# 1) If there are multiple replicas able to failover, they exchange messages -# in order to try to give an advantage to the replica with the best -# replication offset (more data from the master processed). -# Replicas will try to get their rank by offset, and apply to the start -# of the failover a delay proportional to their rank. -# -# 2) Every single replica computes the time of the last interaction with -# its master. This can be the last ping or command received (if the master -# is still in the "connected" state), or the time that elapsed since the -# disconnection with the master (if the replication link is currently down). -# If the last interaction is too old, the replica will not try to failover -# at all. -# -# The point "2" can be tuned by user. Specifically a replica will not perform -# the failover if, since the last interaction with the master, the time -# elapsed is greater than: -# -# (node-timeout * replica-validity-factor) + repl-ping-replica-period -# -# So for example if node-timeout is 30 seconds, and the replica-validity-factor -# is 10, and assuming a default repl-ping-replica-period of 10 seconds, the -# replica will not try to failover if it was not able to talk with the master -# for longer than 310 seconds. -# -# A large replica-validity-factor may allow replicas with too old data to failover -# a master, while a too small value may prevent the cluster from being able to -# elect a replica at all. -# -# For maximum availability, it is possible to set the replica-validity-factor -# to a value of 0, which means, that replicas will always try to failover the -# master regardless of the last time they interacted with the master. -# (However they'll always try to apply a delay proportional to their -# offset rank). -# -# Zero is the only value able to guarantee that when all the partitions heal -# the cluster will always be able to continue. -# -# cluster-replica-validity-factor 10 - -# Cluster replicas are able to migrate to orphaned masters, that are masters -# that are left without working replicas. This improves the cluster ability -# to resist to failures as otherwise an orphaned master can't be failed over -# in case of failure if it has no working replicas. -# -# Replicas migrate to orphaned masters only if there are still at least a -# given number of other working replicas for their old master. This number -# is the "migration barrier". A migration barrier of 1 means that a replica -# will migrate only if there is at least 1 other working replica for its master -# and so forth. It usually reflects the number of replicas you want for every -# master in your cluster. -# -# Default is 1 (replicas migrate only if their masters remain with at least -# one replica). To disable migration just set it to a very large value. -# A value of 0 can be set but is useful only for debugging and dangerous -# in production. -# -# cluster-migration-barrier 1 - -# By default Redis Cluster nodes stop accepting queries if they detect there -# is at least an hash slot uncovered (no available node is serving it). -# This way if the cluster is partially down (for example a range of hash slots -# are no longer covered) all the cluster becomes, eventually, unavailable. -# It automatically returns available as soon as all the slots are covered again. -# -# However sometimes you want the subset of the cluster which is working, -# to continue to accept queries for the part of the key space that is still -# covered. In order to do so, just set the cluster-require-full-coverage -# option to no. -# -# cluster-require-full-coverage yes - -# This option, when set to yes, prevents replicas from trying to failover its -# master during master failures. However the master can still perform a -# manual failover, if forced to do so. -# -# This is useful in different scenarios, especially in the case of multiple -# data center operations, where we want one side to never be promoted if not -# in the case of a total DC failure. -# -# cluster-replica-no-failover no - -# In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. - -########################## CLUSTER DOCKER/NAT support ######################## - -# In certain deployments, Redis Cluster nodes address discovery fails, because -# addresses are NAT-ted or because ports are forwarded (the typical case is -# Docker and other containers). -# -# In order to make Redis Cluster working in such environments, a static -# configuration where each node knows its public address is needed. The -# following two options are used for this scope, and are: -# -# * cluster-announce-ip -# * cluster-announce-port -# * cluster-announce-bus-port -# -# Each instruct the node about its address, client port, and cluster message -# bus port. The information is then published in the header of the bus packets -# so that other nodes will be able to correctly map the address of the node -# publishing the information. -# -# If the above options are not used, the normal Redis Cluster auto-detection -# will be used instead. -# -# Note that when remapped, the bus port may not be at the fixed offset of -# clients port + 10000, so you can specify any port and bus-port depending -# on how they get remapped. If the bus-port is not set, a fixed offset of -# 10000 will be used as usually. -# -# Example: -# -# cluster-announce-ip 10.1.1.5 -# cluster-announce-port 6379 -# cluster-announce-bus-port 6380 - -################################## SLOW LOG ################################### - -# The Redis Slow Log is a system to log queries that exceeded a specified -# execution time. The execution time does not include the I/O operations -# like talking with the client, sending the reply and so forth, -# but just the time needed to actually execute the command (this is the only -# stage of command execution where the thread is blocked and can not serve -# other requests in the meantime). -# -# You can configure the slow log with two parameters: one tells Redis -# what is the execution time, in microseconds, to exceed in order for the -# command to get logged, and the other parameter is the length of the -# slow log. When a new command is logged the oldest one is removed from the -# queue of logged commands. - -# The following time is expressed in microseconds, so 1000000 is equivalent -# to one second. Note that a negative number disables the slow log, while -# a value of zero forces the logging of every command. -slowlog-log-slower-than 10000 - -# There is no limit to this length. Just be aware that it will consume memory. -# You can reclaim memory used by the slow log with SLOWLOG RESET. -slowlog-max-len 128 - -################################ LATENCY MONITOR ############################## - -# The Redis latency monitoring subsystem samples different operations -# at runtime in order to collect data related to possible sources of -# latency of a Redis instance. -# -# Via the LATENCY command this information is available to the user that can -# print graphs and obtain reports. -# -# The system only logs operations that were performed in a time equal or -# greater than the amount of milliseconds specified via the -# latency-monitor-threshold configuration directive. When its value is set -# to zero, the latency monitor is turned off. -# -# By default latency monitoring is disabled since it is mostly not needed -# if you don't have latency issues, and collecting data has a performance -# impact, that while very small, can be measured under big load. Latency -# monitoring can easily be enabled at runtime using the command -# "CONFIG SET latency-monitor-threshold " if needed. -latency-monitor-threshold 0 - -############################# EVENT NOTIFICATION ############################## - -# Redis can notify Pub/Sub clients about events happening in the key space. -# This feature is documented at http://redis.io/topics/notifications -# -# For instance if keyspace events notification is enabled, and a client -# performs a DEL operation on key "foo" stored in the Database 0, two -# messages will be published via Pub/Sub: -# -# PUBLISH __keyspace@0__:foo del -# PUBLISH __keyevent@0__:del foo -# -# It is possible to select the events that Redis will notify among a set -# of classes. Every class is identified by a single character: -# -# K Keyspace events, published with __keyspace@__ prefix. -# E Keyevent events, published with __keyevent@__ prefix. -# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... -# $ String commands -# l List commands -# s Set commands -# h Hash commands -# z Sorted set commands -# x Expired events (events generated every time a key expires) -# e Evicted events (events generated when a key is evicted for maxmemory) -# A Alias for g$lshzxe, so that the "AKE" string means all the events. -# -# The "notify-keyspace-events" takes as argument a string that is composed -# of zero or multiple characters. The empty string means that notifications -# are disabled. -# -# Example: to enable list and generic events, from the point of view of the -# event name, use: -# -# notify-keyspace-events Elg -# -# Example 2: to get the stream of the expired keys subscribing to channel -# name __keyevent@0__:expired use: -# -# notify-keyspace-events Ex -# -# By default all notifications are disabled because most users don't need -# this feature and the feature has some overhead. Note that if you don't -# specify at least one of K or E, no events will be delivered. -notify-keyspace-events "" - -############################### ADVANCED CONFIG ############################### - -# Hashes are encoded using a memory efficient data structure when they have a -# small number of entries, and the biggest entry does not exceed a given -# threshold. These thresholds can be configured using the following directives. -hash-max-ziplist-entries 512 -hash-max-ziplist-value 64 - -# Lists are also encoded in a special way to save a lot of space. -# The number of entries allowed per internal list node can be specified -# as a fixed maximum size or a maximum number of elements. -# For a fixed maximum size, use -5 through -1, meaning: -# -5: max size: 64 Kb <-- not recommended for normal workloads -# -4: max size: 32 Kb <-- not recommended -# -3: max size: 16 Kb <-- probably not recommended -# -2: max size: 8 Kb <-- good -# -1: max size: 4 Kb <-- good -# Positive numbers mean store up to _exactly_ that number of elements -# per list node. -# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), -# but if your use case is unique, adjust the settings as necessary. -list-max-ziplist-size -2 - -# Lists may also be compressed. -# Compress depth is the number of quicklist ziplist nodes from *each* side of -# the list to *exclude* from compression. The head and tail of the list -# are always uncompressed for fast push/pop operations. Settings are: -# 0: disable all list compression -# 1: depth 1 means "don't start compressing until after 1 node into the list, -# going from either the head or tail" -# So: [head]->node->node->...->node->[tail] -# [head], [tail] will always be uncompressed; inner nodes will compress. -# 2: [head]->[next]->node->node->...->node->[prev]->[tail] -# 2 here means: don't compress head or head->next or tail->prev or tail, -# but compress all nodes between them. -# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] -# etc. -list-compress-depth 0 - -# Sets have a special encoding in just one case: when a set is composed -# of just strings that happen to be integers in radix 10 in the range -# of 64 bit signed integers. -# The following configuration setting sets the limit in the size of the -# set in order to use this special memory saving encoding. -set-max-intset-entries 512 - -# Similarly to hashes and lists, sorted sets are also specially encoded in -# order to save a lot of space. This encoding is only used when the length and -# elements of a sorted set are below the following limits: -zset-max-ziplist-entries 128 -zset-max-ziplist-value 64 - -# HyperLogLog sparse representation bytes limit. The limit includes the -# 16 bytes header. When an HyperLogLog using the sparse representation crosses -# this limit, it is converted into the dense representation. -# -# A value greater than 16000 is totally useless, since at that point the -# dense representation is more memory efficient. -# -# The suggested value is ~ 3000 in order to have the benefits of -# the space efficient encoding without slowing down too much PFADD, -# which is O(N) with the sparse encoding. The value can be raised to -# ~ 10000 when CPU is not a concern, but space is, and the data set is -# composed of many HyperLogLogs with cardinality in the 0 - 15000 range. -hll-sparse-max-bytes 3000 - -# Streams macro node max size / items. The stream data structure is a radix -# tree of big nodes that encode multiple items inside. Using this configuration -# it is possible to configure how big a single node can be in bytes, and the -# maximum number of items it may contain before switching to a new node when -# appending new stream entries. If any of the following settings are set to -# zero, the limit is ignored, so for instance it is possible to set just a -# max entires limit by setting max-bytes to 0 and max-entries to the desired -# value. -stream-node-max-bytes 4096 -stream-node-max-entries 100 - -# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in -# order to help rehashing the main Redis hash table (the one mapping top-level -# keys to values). The hash table implementation Redis uses (see dict.c) -# performs a lazy rehashing: the more operation you run into a hash table -# that is rehashing, the more rehashing "steps" are performed, so if the -# server is idle the rehashing is never complete and some more memory is used -# by the hash table. -# -# The default is to use this millisecond 10 times every second in order to -# actively rehash the main dictionaries, freeing memory when possible. -# -# If unsure: -# use "activerehashing no" if you have hard latency requirements and it is -# not a good thing in your environment that Redis can reply from time to time -# to queries with 2 milliseconds delay. -# -# use "activerehashing yes" if you don't have such hard requirements but -# want to free memory asap when possible. -activerehashing yes - -# The client output buffer limits can be used to force disconnection of clients -# that are not reading data from the server fast enough for some reason (a -# common reason is that a Pub/Sub client can't consume messages as fast as the -# publisher can produce them). -# -# The limit can be set differently for the three different classes of clients: -# -# normal -> normal clients including MONITOR clients -# replica -> replica clients -# pubsub -> clients subscribed to at least one pubsub channel or pattern -# -# The syntax of every client-output-buffer-limit directive is the following: -# -# client-output-buffer-limit -# -# A client is immediately disconnected once the hard limit is reached, or if -# the soft limit is reached and remains reached for the specified number of -# seconds (continuously). -# So for instance if the hard limit is 32 megabytes and the soft limit is -# 16 megabytes / 10 seconds, the client will get disconnected immediately -# if the size of the output buffers reach 32 megabytes, but will also get -# disconnected if the client reaches 16 megabytes and continuously overcomes -# the limit for 10 seconds. -# -# By default normal clients are not limited because they don't receive data -# without asking (in a push way), but just after a request, so only -# asynchronous clients may create a scenario where data is requested faster -# than it can read. -# -# Instead there is a default limit for pubsub and replica clients, since -# subscribers and replicas receive data in a push fashion. -# -# Both the hard or the soft limit can be disabled by setting them to zero. -client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit replica 256mb 64mb 60 -client-output-buffer-limit pubsub 32mb 8mb 60 - -# Client query buffers accumulate new commands. They are limited to a fixed -# amount by default in order to avoid that a protocol desynchronization (for -# instance due to a bug in the client) will lead to unbound memory usage in -# the query buffer. However you can configure it here if you have very special -# needs, such us huge multi/exec requests or alike. -# -# client-query-buffer-limit 1gb - -# In the Redis protocol, bulk requests, that are, elements representing single -# strings, are normally limited ot 512 mb. However you can change this limit -# here. -# -# proto-max-bulk-len 512mb - -# Redis calls an internal function to perform many background tasks, like -# closing connections of clients in timeout, purging expired keys that are -# never requested, and so forth. -# -# Not all tasks are performed with the same frequency, but Redis checks for -# tasks to perform according to the specified "hz" value. -# -# By default "hz" is set to 10. Raising the value will use more CPU when -# Redis is idle, but at the same time will make Redis more responsive when -# there are many keys expiring at the same time, and timeouts may be -# handled with more precision. -# -# The range is between 1 and 500, however a value over 100 is usually not -# a good idea. Most users should use the default of 10 and raise this up to -# 100 only in environments where very low latency is required. -hz 10 - -# Normally it is useful to have an HZ value which is proportional to the -# number of clients connected. This is useful in order, for instance, to -# avoid too many clients are processed for each background task invocation -# in order to avoid latency spikes. -# -# Since the default HZ value by default is conservatively set to 10, Redis -# offers, and enables by default, the ability to use an adaptive HZ value -# which will temporary raise when there are many connected clients. -# -# When dynamic HZ is enabled, the actual configured HZ will be used as -# as a baseline, but multiples of the configured HZ value will be actually -# used as needed once more clients are connected. In this way an idle -# instance will use very little CPU time while a busy instance will be -# more responsive. -dynamic-hz yes - -# When a child rewrites the AOF file, if the following option is enabled -# the file will be fsync-ed every 32 MB of data generated. This is useful -# in order to commit the file to the disk more incrementally and avoid -# big latency spikes. -aof-rewrite-incremental-fsync yes - -# When redis saves RDB file, if the following option is enabled -# the file will be fsync-ed every 32 MB of data generated. This is useful -# in order to commit the file to the disk more incrementally and avoid -# big latency spikes. -rdb-save-incremental-fsync yes - -# Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good -# idea to start with the default settings and only change them after investigating -# how to improve the performances and how the keys LFU change over time, which -# is possible to inspect via the OBJECT FREQ command. -# -# There are two tunable parameters in the Redis LFU implementation: the -# counter logarithm factor and the counter decay time. It is important to -# understand what the two parameters mean before changing them. -# -# The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis -# uses a probabilistic increment with logarithmic behavior. Given the value -# of the old counter, when a key is accessed, the counter is incremented in -# this way: -# -# 1. A random number R between 0 and 1 is extracted. -# 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). -# 3. The counter is incremented only if R < P. -# -# The default lfu-log-factor is 10. This is a table of how the frequency -# counter changes with a different number of accesses with different -# logarithmic factors: -# -# +--------+------------+------------+------------+------------+------------+ -# | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | -# +--------+------------+------------+------------+------------+------------+ -# | 0 | 104 | 255 | 255 | 255 | 255 | -# +--------+------------+------------+------------+------------+------------+ -# | 1 | 18 | 49 | 255 | 255 | 255 | -# +--------+------------+------------+------------+------------+------------+ -# | 10 | 10 | 18 | 142 | 255 | 255 | -# +--------+------------+------------+------------+------------+------------+ -# | 100 | 8 | 11 | 49 | 143 | 255 | -# +--------+------------+------------+------------+------------+------------+ -# -# NOTE: The above table was obtained by running the following commands: -# -# redis-benchmark -n 1000000 incr foo -# redis-cli object freq foo -# -# NOTE 2: The counter initial value is 5 in order to give new objects a chance -# to accumulate hits. -# -# The counter decay time is the time, in minutes, that must elapse in order -# for the key counter to be divided by two (or decremented if it has a value -# less <= 10). -# -# The default value for the lfu-decay-time is 1. A Special value of 0 means to -# decay the counter every time it happens to be scanned. -# -# lfu-log-factor 10 -# lfu-decay-time 1 - -########################### ACTIVE DEFRAGMENTATION ####################### -# -# WARNING THIS FEATURE IS EXPERIMENTAL. However it was stress tested -# even in production and manually tested by multiple engineers for some -# time. -# -# What is active defragmentation? -# ------------------------------- -# -# Active (online) defragmentation allows a Redis server to compact the -# spaces left between small allocations and deallocations of data in memory, -# thus allowing to reclaim back memory. -# -# Fragmentation is a natural process that happens with every allocator (but -# less so with Jemalloc, fortunately) and certain workloads. Normally a server -# restart is needed in order to lower the fragmentation, or at least to flush -# away all the data and create it again. However thanks to this feature -# implemented by Oran Agra for Redis 4.0 this process can happen at runtime -# in an "hot" way, while the server is running. -# -# Basically when the fragmentation is over a certain level (see the -# configuration options below) Redis will start to create new copies of the -# values in contiguous memory regions by exploiting certain specific Jemalloc -# features (in order to understand if an allocation is causing fragmentation -# and to allocate it in a better place), and at the same time, will release the -# old copies of the data. This process, repeated incrementally for all the keys -# will cause the fragmentation to drop back to normal values. -# -# Important things to understand: -# -# 1. This feature is disabled by default, and only works if you compiled Redis -# to use the copy of Jemalloc we ship with the source code of Redis. -# This is the default with Linux builds. -# -# 2. You never need to enable this feature if you don't have fragmentation -# issues. -# -# 3. Once you experience fragmentation, you can enable this feature when -# needed with the command "CONFIG SET activedefrag yes". -# -# The configuration parameters are able to fine tune the behavior of the -# defragmentation process. If you are not sure about what they mean it is -# a good idea to leave the defaults untouched. - -# Enabled active defragmentation -# activedefrag yes - -# Minimum amount of fragmentation waste to start active defrag -# active-defrag-ignore-bytes 100mb - -# Minimum percentage of fragmentation to start active defrag -# active-defrag-threshold-lower 10 - -# Maximum percentage of fragmentation at which we use maximum effort -# active-defrag-threshold-upper 100 - -# Minimal effort for defrag in CPU percentage -# active-defrag-cycle-min 5 - -# Maximal effort for defrag in CPU percentage -# active-defrag-cycle-max 75 - -# Maximum number of set/hash/zset/list fields that will be processed from -# the main dictionary scan -# active-defrag-max-scan-fields 1000 diff --git a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema index eb09fa627..d51b9c1b2 100644 --- a/apps/emqx_auth_redis/priv/emqx_auth_redis.schema +++ b/apps/emqx_auth_redis/priv/emqx_auth_redis.schema @@ -33,6 +33,40 @@ hidden ]}. +{mapping, "auth.redis.ssl", "emqx_auth_redis.options", [ + {default, off}, + {datatype, flag} +]}. + +{mapping, "auth.redis.cafile", "emqx_auth_redis.options", [ + {default, ""}, + {datatype, string} +]}. + +{mapping, "auth.redis.certfile", "emqx_auth_redis.options", [ + {default, ""}, + {datatype, string} +]}. + +{mapping, "auth.redis.keyfile", "emqx_auth_redis.options", [ + {default, ""}, + {datatype, string} +]}. + +{translation, "emqx_auth_redis.options", fun(Conf) -> + Ssl = cuttlefish:conf_get("auth.redis.ssl", Conf, false), + case Ssl of + true -> + CA = cuttlefish:conf_get("auth.redis.cafile", Conf), + Cert = cuttlefish:conf_get("auth.redis.certfile", Conf), + Key = cuttlefish:conf_get("auth.redis.keyfile", Conf), + [{options, [{ssl_options, [{cacertfile, CA}, + {certfile, Cert}, + {keyfile, Key}]}]}]; + _ -> [{options, []}] + end +end}. + {translation, "emqx_auth_redis.server", fun(Conf) -> Fun = fun(S) -> case string:split(S, ":", trailing) of @@ -103,4 +137,3 @@ end}. _ -> plain end end}. - diff --git a/apps/emqx_auth_redis/rebar.config b/apps/emqx_auth_redis/rebar.config index dfdf44cc2..af88d3665 100644 --- a/apps/emqx_auth_redis/rebar.config +++ b/apps/emqx_auth_redis/rebar.config @@ -1,3 +1,31 @@ {deps, - [{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "v0.6.1"}}} - ]}. \ No newline at end of file + [{eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.2"}}} + ]}. + +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + compressed + ]}. +{overrides, [{add, [{erl_opts, [compressed]}]}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions + ]}. + +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helper", {tag, "1.2.2"}}}, + {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.0"}}} + ]}, + {erl_opts, [debug_info]} + ]} + ]}. diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_cli.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_cli.erl index c6d5dea69..31cb67505 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis_cli.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_cli.erl @@ -41,13 +41,13 @@ connect(Opts) -> eredis_sentinel:start_link(get_value(servers, Opts)), "sentinel:" ++ Sentinel end, - case eredis:start_link( - Host, - get_value(port, Opts, 6379), - get_value(database, Opts), - get_value(password, Opts), - no_reconnect - ) of + case eredis:start_link(Host, + get_value(port, Opts, 6379), + get_value(database, Opts, 0), + get_value(password, Opts, ""), + 3000, + 5000, + get_value(options, Opts, [])) of {ok, Pid} -> {ok, Pid}; {error, Reason = {connection_error, _}} -> ?LOG(error, "[Redis] Can't connect to Redis server: Connection refused."), diff --git a/apps/emqx_auth_redis/src/emqx_auth_redis_sup.erl b/apps/emqx_auth_redis/src/emqx_auth_redis_sup.erl index 41148c88e..6066a306a 100644 --- a/apps/emqx_auth_redis/src/emqx_auth_redis_sup.erl +++ b/apps/emqx_auth_redis/src/emqx_auth_redis_sup.erl @@ -32,11 +32,12 @@ init([]) -> {ok, {{one_for_one, 10, 100}, pool_spec(Server)}}. pool_spec(Server) -> + Options = application:get_env(?APP, options, []), case proplists:get_value(type, Server) of cluster -> - eredis_cluster:start_pool(?APP, Server), + eredis_cluster:start_pool(?APP, Server ++ Options), []; _ -> - [ecpool:pool_spec(?APP, ?APP, emqx_auth_redis_cli, Server)] + [ecpool:pool_spec(?APP, ?APP, emqx_auth_redis_cli, Server ++ Options)] end. diff --git a/apps/emqx_bridge_mqtt/rebar.config b/apps/emqx_bridge_mqtt/rebar.config index 7b1f06cca..1958724ac 100644 --- a/apps/emqx_bridge_mqtt/rebar.config +++ b/apps/emqx_bridge_mqtt/rebar.config @@ -1 +1,27 @@ -{deps, []}. \ No newline at end of file +{deps, []}. +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{shell, [ + % {config, "config/sys.config"}, + {apps, [emqx, emqx_bridge_mqtt]} +]}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}}]} + ]} +]}. diff --git a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl index ca3bba095..e46c6d05b 100644 --- a/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl +++ b/apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl @@ -556,7 +556,7 @@ default => <<"">>, title => #{en => <<"Payload Template">>, zh => <<"消息内容模板"/utf8>>}, - description => #{en => <<"The payload template, variable interpolation is supported. If using empty template (default), then the payload will be all the available vars in JOSN format">>, + description => #{en => <<"The payload template, variable interpolation is supported. If using empty template (default), then the payload will be all the available vars in JSON format">>, zh => <<"消息内容模板，支持变量。若使用空模板（默认），消息内容为 JSON 格式的所有字段"/utf8>>} } }, diff --git a/apps/emqx_coap/rebar.config b/apps/emqx_coap/rebar.config index 655e3b14f..dd1ad613e 100644 --- a/apps/emqx_coap/rebar.config +++ b/apps/emqx_coap/rebar.config @@ -1,4 +1,28 @@ {deps, [ {gen_coap, {git, "https://github.com/emqx/gen_coap", {tag, "v0.3.0"}}} - ]}. \ No newline at end of file + ]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {parse_transform}]}. + +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. + +{profiles, + [{test, + [{deps, + [{er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}, + {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.2.2"}}} + ]} + ]} + ]}. diff --git a/apps/emqx_dashboard/rebar.config b/apps/emqx_dashboard/rebar.config index 7b1f06cca..5d69c8c0e 100644 --- a/apps/emqx_dashboard/rebar.config +++ b/apps/emqx_dashboard/rebar.config @@ -1 +1,23 @@ -{deps, []}. \ No newline at end of file +{deps, []}. + +{profiles, + [{test, + [{deps, + [{emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helper", {branch, "1.2.2"}}} + ]} + ]} + ]}. + +{edoc_opts, [{preprocess, true}]}. +{erl_opts, [warn_unused_vars, + warn_shadow_vars, + warn_unused_import, + warn_obsolete_guard, + debug_info, + {d, 'APPLICATION', emqx}]}. +{xref_checks, [undefined_function_calls, undefined_functions, + locals_not_used, deprecated_function_calls, + warnings_as_errors, deprecated_functions]}. +{cover_enabled, true}. +{cover_opts, [verbose]}. +{cover_export_enabled, true}. diff --git a/apps/emqx_exhook/.gitignore b/apps/emqx_exhook/.gitignore deleted file mode 100644 index 520db35b8..000000000 --- a/apps/emqx_exhook/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -src/emqx_exhook_pb.erl -src/emqx_exhook_v_1_hook_provider_bhvr.erl -src/emqx_exhook_v_1_hook_provider_client.erl diff --git a/apps/emqx_exhook/README.md b/apps/emqx_exhook/README.md index 216c39275..9d4ccd81f 100644 --- a/apps/emqx_exhook/README.md +++ b/apps/emqx_exhook/README.md @@ -1,39 +1,75 @@ -# emqx_exhook +# emqx_extension_hook -The `emqx_exhook` extremly enhance the extensibility for EMQ X. It allow using an others programming language to mount the hooks intead of erlang. +The `emqx_extension_hook` extremly enhance the extensibility for EMQ X. It allow using an others programming language to mount the hooks intead of erlang. ## Feature -- [x] Based on gRPC, it brings a very wide range of applicability +- [x] Support `python` and `java`. +- [x] Support all hooks of emqx. - [x] Allows you to use the return value to extend emqx behavior. +We temporarily no plans to support other languages. Plaease open a issue if you have to use other programming languages. + ## Architecture ``` -EMQ X Third-party Runtime -+========================+ +========+==========+ -| ExHook | | | | -| +----------------+ | gRPC | gRPC | User's | -| | gPRC Client | ------------------> | Server | Codes | -| +----------------+ | (HTTP/2) | | | -| | | | | -+========================+ +========+==========+ + EMQ X Third-party Runtimes ++========================+ +====================+ +| Extension | | | +| +----------------+ | Hooks | Python scripts / | +| | Drivers | ------------------> | Java Classes / | +| +----------------+ | (pipe) | Others ... | +| | | | ++========================+ +====================+ ``` -## Usage +## Drivers -### gRPC service +### Python -See: `priv/protos/exhook.proto` +***Requirements:*** -### CLI +- It requires the emqx hosted machine has Python3 Runtimes (not support python2) +- The `python3` executable commands in your shell -## Example +***Examples:*** -## Recommended gRPC Framework +See `test/scripts/main.py` -See: https://github.com/grpc-ecosystem/awesome-grpc +### Java -## Thanks +***Requirements:*** -- [grpcbox](https://github.com/tsloughter/grpcbox) +- It requires the emqx hosted machine has Java 8+ Runtimes +- An executable commands in your shell, i,g: `java` + +***Examples:*** + +See `test/scripts/Main.java` + +## Configurations + +| Name | Data Type | Options | Default | Description | +| ------------------- | --------- | ------------------------------------- | ---------------- | -------------------------------- | +| drivers | Enum | `python3`
`java` | `python3` | Drivers type | +| .path | String | - | `data/extension` | The codes/library search path | +| .call_timeout | Duration | - | `5s` | Function call timeout | +| .pool_size | Integer | - | `8` | The pool size for the driver | +| .init_module | String | - | main | The module name for initial call | + +## SDK + +See `sdk/README.md` + +## Known Issues or TODOs + +**Configurable Log System** + +- use stderr to print logs to the emqx console instead of stdout. An alternative is to print the logs to a file. +- The Java driver can not redirect the `stderr` stream to erlang vm on Windows platform. + +## Reference + +- [erlport](https://github.com/hdima/erlport) +- [Eexternal Term Format](http://erlang.org/doc/apps/erts/erl_ext_dist.html) +- [The Ports Tutorial of Erlang](http://erlang.org/doc/tutorial/c_port.html) diff --git a/apps/emqx_exhook/docs/design.md b/apps/emqx_exhook/docs/design.md index 671e240cc..1bf74723c 100644 --- a/apps/emqx_exhook/docs/design.md +++ b/apps/emqx_exhook/docs/design.md @@ -2,115 +2,254 @@ ## 动机 -在 EMQ X Broker v4.1-v4.2 中，我们发布了 2 个插件来扩展 emqx 的编程能力： +增强系统的扩展性。包含的目的有： -1. `emqx-extension-hook` 提供了使用 Java, Python 向 Broker 挂载钩子的功能 -2. `emqx-exproto` 提供了使用 Java，Python 编写用户自定义协议接入插件的功能 +- 完全支持各种钩子，能够根据其返回值修改 EMQ X 或者 Client 的行为。 + - 例如 `auth/acl`：可以查询数据库或者执行某种算法校验操作权限。然后返回 `false` 表示 `认证/ACL` 失败。 + - 例如 `message.publish`：可以解析 `消息/主题` 并将其存储至数据库中。 -但在后续的支持中发现许多难以处理的问题： +- 支持多种语言的扩展；并包含该语言的示例程序。 + - python + - webhook + - Java + - Lua + - c，go，..... +- 热操作 + - 允许在插件运行过程中，添加和移除 `Driver`。 -1. 有大量的编程语言需要支持，需要编写和维护如 Go, JavaScript, Lua.. 等语言的驱动。 -2. `erlport` 使用的操作系统的管道进行通信，这让用户代码只能部署在和 emqx 同一个操作系统上。部署方式受到了极大的限制。 -3. 用户程序的启动参数直接打包到 Broker 中，导致用户开发无法实时的进行调试，单步跟踪等。 -4. `erlport` 会占用 `stdin` `stdout`。 +- 需要 CLI ，甚至 API 来管理 `Driver` -因此，我们计划重构这部分的实现，其中主要的内容是： -1. 使用 `gRPC` 替换 `erlport`。 -2. 将 `emqx-extension-hook` 重命名为 `emqx-exhook` - - -旧版本的设计参考：[emqx-extension-hook design in v4.2.0](https://github.com/emqx/emqx-exhook/blob/v4.2.0/docs/design.md) +注：`message` 类钩子仅包括在企业版中。 ## 设计 架构如下： ``` - EMQ X -+========================+ +========+==========+ -| ExHook | | | | -| +----------------+ | gRPC | gRPC | User's | -| | gRPC Client | ------------------> | Server | Codes | -| +----------------+ | (HTTP/2) | | | -| | | | | -+========================+ +========+==========+ -``` - -`emqx-exhook` 通过 gRPC 的方式向用户部署的 gRPC 服务发送钩子的请求，并处理其返回的值。 - - -和 emqx 原生的钩子一致，emqx-exhook 也支持链式的方式计算和返回： - - - -### gRPC 服务示例 - -用户需要实现的方法，和数据类型的定义在 `priv/protos/exhook.proto` 文件中。例如，其支持的接口有： - -```protobuff -syntax = "proto3"; - -package emqx.exhook.v1; - -service HookProvider { - - rpc OnProviderLoaded(ProviderLoadedRequest) returns (LoadedResponse) {}; - - rpc OnProviderUnloaded(ProviderUnloadedRequest) returns (EmptySuccess) {}; - - rpc OnClientConnect(ClientConnectRequest) returns (EmptySuccess) {}; - - rpc OnClientConnack(ClientConnackRequest) returns (EmptySuccess) {}; - - rpc OnClientConnected(ClientConnectedRequest) returns (EmptySuccess) {}; - - rpc OnClientDisconnected(ClientDisconnectedRequest) returns (EmptySuccess) {}; - - rpc OnClientAuthenticate(ClientAuthenticateRequest) returns (ValuedResponse) {}; - - rpc OnClientCheckAcl(ClientCheckAclRequest) returns (ValuedResponse) {}; - - rpc OnClientSubscribe(ClientSubscribeRequest) returns (EmptySuccess) {}; - - rpc OnClientUnsubscribe(ClientUnsubscribeRequest) returns (EmptySuccess) {}; - - rpc OnSessionCreated(SessionCreatedRequest) returns (EmptySuccess) {}; - - rpc OnSessionSubscribed(SessionSubscribedRequest) returns (EmptySuccess) {}; - - rpc OnSessionUnsubscribed(SessionUnsubscribedRequest) returns (EmptySuccess) {}; - - rpc OnSessionResumed(SessionResumedRequest) returns (EmptySuccess) {}; - - rpc OnSessionDiscarded(SessionDiscardedRequest) returns (EmptySuccess) {}; - - rpc OnSessionTakeovered(SessionTakeoveredRequest) returns (EmptySuccess) {}; - - rpc OnSessionTerminated(SessionTerminatedRequest) returns (EmptySuccess) {}; - - rpc OnMessagePublish(MessagePublishRequest) returns (ValuedResponse) {}; - - rpc OnMessageDelivered(MessageDeliveredRequest) returns (EmptySuccess) {}; - - rpc OnMessageDropped(MessageDroppedRequest) returns (EmptySuccess) {}; - - rpc OnMessageAcked(MessageAckedRequest) returns (EmptySuccess) {}; -} + EMQ X Third-party Runtimes ++========================+ +====================+ +| Extension | | | +| +----------------+ | Hooks | Python scripts / | +| | Drivers | ------------------> | Java Classes / | +| +----------------+ | (pipe) | Others ... | +| | | | ++========================+ +====================+ ``` ### 配置文件示例 -``` -## 配置 gRPC 服务地址 (HTTP) -## -## s1 为服务器的名称 -exhook.server.s1.url = http://127.0.0.1:9001 +#### 驱动 配置 -## 配置 gRPC 服务地址 (HTTPS) +```properties +## Driver type ## -## s2 为服务器名称 -exhook.server.s2.url = https://127.0.0.1:9002 -exhook.server.s2.cacertfile = ca.pem -exhook.server.s2.certfile = cert.pem -exhook.server.s2.keyfile = key.pem +## Exmaples: +## - python3 --- 仅配置 python3 +## - python3, java, webhook --- 配置多个 Driver +exhook.dirvers = python3, java, webhook + +## --- 具体 driver 的配置详情 + +## Python +exhook.dirvers.python3.path = data/extension/python +exhook.dirvers.python3.call_timeout = 5s +exhook.dirvers.python3.pool_size = 8 + +## java +exhook.drivers.java.path = data/extension/java +... +``` + +#### 钩子配置 + +钩子支持配置在配置文件中，例如： + +```properties +exhook.rule.python3.client.connected = {"module": "client", "callback": "on_client_connected"} +exhook.rule.python3.message.publish = {"module": "client", "callback": "on_client_connected", "topics": ["#", "t/#"]} +``` + +***已废弃！！（冗余）*** + + +### 驱动抽象 + +#### APIs + +| 方法名 | 说明 | 入参 | 返回 | +| ------------------------ | -------- | ------ | ------ | +| `init` | 初始化 | - | 见下表 | +| `deinit` | 销毁 | - | - | +| `xxx `*(由init函数定义)* | 钩子回调 | 见下表 | 见下表 | + + + +##### init 函数规格 + +```erlang +%% init 函数 +%% HookSpec : 为用户在脚本中的 初始化函数指定的；他会与配置文件中的内容作为默认值，进行合并 +%% 该参数的目的，用于 EMQ X 判断需要执行哪些 Hook 和 如何执行 Hook +%% State : 为用户自己管理的数据内容，EMQ X 不关心它，只来回透传 +init() -> {HookSpec, State}. + +%% 例如： +{[{client_connect, callback_m(), callback_f(),#{}, {}}]} + +%%-------------------------------------------------------------- +%% Type Defines + +-tpye hook_spec() :: [{hookname(), callback_m(), callback_f(), hook_opts()}]. + +-tpye state :: any(). + +-type hookname() :: client_connect + | client_connack + | client_connected + | client_disconnected + | client_authenticate + | client_check_acl + | client_subscribe + | client_unsubscribe + | session_created + | session_subscribed + | session_unsubscribed + | session_resumed + | session_discarded %% TODO: Should squash to `terminated` ? + | session_takeovered %% TODO: Should squash to `terminated` ? + | session_terminated + | message_publish + | message_delivered + | message_acked + | message_dropped. + +-type callback_m() :: atom(). -- 回调的模块名称；python 为脚本文件名称；java 为类名；webhook 为 URI 地址 + +-type callback_f() :: atom(). -- 回调的方法名称；python，java 等为方法名；webhook 为资源地址 + +-tpye hook_opts() :: [{hook_key(), any()}]. -- 配置项；配置该项钩子的行为 + +-type hook_key() :: topics | ... +``` + + + +##### deinit 函数规格 + +``` erlang +%% deinit 函数；不关心返回的任何内容 +deinit() -> any(). +``` + + + +##### 回调函数规格 + +| 钩子 | 入参 | 返回 | +| -------------------- | ----------------------------------------------------- | --------- | +| client_connect | `connifno`
`props` | - | +| client_connack | `connifno`
`rc`
`props` | - | +| client_connected | `clientinfo`
| - | +| client_disconnected | `clientinfo`
`reason` | - | +| client_authenticate | `clientinfo`
`result` | `result` | +| client_check_acl | `clientinfo`
`pubsub`
`topic`
`result` | `result` | +| client_subscribe | `clientinfo`
`props`
`topicfilters` | - | +| client_unsubscribe | `clientinfo`
`props`
`topicfilters` | - | +| session_created | `clientinfo` | - | +| session_subscribed | `clientinfo`
`topic`
`subopts` | - | +| session_unsubscribed | `clientinfo`
`topic` | - | +| session_resumed | `clientinfo` | - | +| session_discared | `clientinfo` | - | +| session_takeovered | `clientinfo` | - | +| session_terminated | `clientinfo`
`reason` | - | +| message_publish | `messsage` | `message` | +| message_delivered | `clientinfo`
`message` | - | +| message_dropped | `message` | - | +| message_acked | `clientinfo`
`message` | - | + + + +上表中包含数据格式为： + +```erlang +-type conninfo :: [ {node, atom()} + , {clientid, binary()} + , {username, binary()} + , {peerhost, binary()} + , {sockport, integer()} + , {proto_name, binary()} + , {proto_ver, integer()} + , {keepalive, integer()} + ]. + +-type clientinfo :: [ {node, atom()} + , {clientid, binary()} + , {username, binary()} + , {password, binary()} + , {peerhost, binary()} + , {sockport, integer()} + , {protocol, binary()} + , {mountpoint, binary()} + , {is_superuser, boolean()} + , {anonymous, boolean()} + ]. + +-type message :: [ {node, atom()} + , {id, binary()} + , {qos, integer()} + , {from, binary()} + , {topic, binary()} + , {payload, binary()} + , {timestamp, integer()} + ]. + +-type rc :: binary(). +-type props :: [{key(), value()}] + +-type topics :: [topic()]. +-type topic :: binary(). +-type pubsub :: publish | subscribe. +-type result :: true | false. +``` + + + +### 统计 + +在驱动运行过程中，应有对每种钩子调用计数，例如： + +``` +exhook.python3.check_acl 10 +``` + + + +### 管理 + +**CLI 示例：** + + + +**列出所有的驱动** + +``` +./bin/emqx_ctl exhook dirvers list +Drivers(xxx=yyy) +Drivers(aaa=bbb) +``` + + + +**开关驱动** + +``` +./bin/emqx_ctl exhook drivers enable python3 +ok + +./bin/emqx_ctl exhook drivers disable python3 +ok + +./bin/emqx_ctl exhook drivers stats +python3.client_connect 123 +webhook.check_acl 20 ``` diff --git a/apps/emqx_exhook/rebar.config b/apps/emqx_exhook/rebar.config index 1db04037f..7d5a285fb 100644 --- a/apps/emqx_exhook/rebar.config +++ b/apps/emqx_exhook/rebar.config @@ -1,21 +1,5 @@ %%-*- mode: erlang -*- -{plugins, - [rebar3_proper, - {grpcbox_plugin, {git, "https://github.com/zmstone/grpcbox_plugin", {branch, "master"}}} -]}. - -{deps, - [{grpcbox, {git, "https://github.com/tsloughter/grpcbox", {branch, "master"}}} -]}. - -{grpc, - [{protos, ["priv/protos"]}, - {gpb_opts, [{module_name_prefix, "emqx_"}, - {module_name_suffix, "_pb"}]} -]}. - -{provider_hooks, - [{pre, [{compile, {grpc, gen}}]}]}. +{deps, []}. {edoc_opts, [{preprocess, true}]}. @@ -29,18 +13,13 @@ {xref_checks, [undefined_function_calls, undefined_functions, locals_not_used, deprecated_function_calls, warnings_as_errors, deprecated_functions]}. -{xref_ignores, [emqx_exhook_pb]}. - {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. -{cover_excl_mods, [emqx_exhook_pb, - emqx_exhook_v_1_hook_provider_bhvr, - emqx_exhook_v_1_connection_client]}. {profiles, [{test, [ - {deps, [ {emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.3.1"}}} + {deps, [ {emqx_ct_helper, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "v1.2.2"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} ]} ]} diff --git a/apps/emqx_exproto/.gitignore b/apps/emqx_exproto/.gitignore deleted file mode 100644 index 791a6e94e..000000000 --- a/apps/emqx_exproto/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -src/emqx_exproto_pb.erl -src/emqx_exproto_v_1_connection_adapter_bhvr.erl -src/emqx_exproto_v_1_connection_adapter_client.erl -src/emqx_exproto_v_1_connection_handler_bhvr.erl -src/emqx_exproto_v_1_connection_handler_client.erl diff --git a/apps/emqx_exproto/README.md b/apps/emqx_exproto/README.md index 4b59dcae3..7fa88a9dc 100644 --- a/apps/emqx_exproto/README.md +++ b/apps/emqx_exproto/README.md @@ -4,25 +4,53 @@ The `emqx_exproto` extremly enhance the extensibility for EMQ X. It allow using ## Feature -- [x] Based on gRPC, it brings a very wide range of applicability -- [x] Allows you to use the return value to extend emqx behavior. +- [x] Support Python, Java. +- [x] Support the `tcp`, `ssl`, `udp`, `dtls` socket. +- [x] Provide the `PUB/SUB` interface to others language. + +We temporarily no plans to support other languages. Plaease open a issue if you have to use other programming languages. ## Architecture ![EMQ X ExProto Arch](./docs/images/exproto-arch.jpg) -## Usage +## Drivers -### gRPC service +### Python -See: `priv/protos/exproto.proto` +***Requirements:*** -## Example +- It requires the emqx hosted machine has Python3 Runtimes +- An executable commands in your shell, i,g: `python3` or `python` -## Recommended gRPC Framework +***Examples:*** -See: https://github.com/grpc-ecosystem/awesome-grpc +See [example/main.python](https://github.com/emqx/emqx-exproto/blob/master/example/main.py) -## Thanks +### Java -- [grpcbox](https://github.com/tsloughter/grpcbox) +See [example/Main.java](https://github.com/emqx/emqx-exproto/blob/master/example/Main.java) + + +## SDK + +The SDK encloses the underlying obscure data types and function interfaces. It only provides a convenience for development, it is not required. + +See [sdk/README.md](https://github.com/emqx/emqx-exproto/blob/master/sdk/README.md) + + +## Benchmark + +***Work in progress...*** + + +## Known Issues or TODOs + +- Configurable Log System. + * The Java driver can not redirect the `stderr` stream to erlang vm on Windows platform + +## Reference + +- [erlport](https://github.com/hdima/erlport) +- [External Term Format](http://erlang.org/doc/apps/erts/erl_ext_dist.html) +- [The Ports Tutorial of Erlang](http://erlang.org/doc/tutorial/c_port.html) diff --git a/apps/emqx_exproto/docs/design.md b/apps/emqx_exproto/docs/design.md index 0a6a082e2..b5cc4e49d 100644 --- a/apps/emqx_exproto/docs/design.md +++ b/apps/emqx_exproto/docs/design.md @@ -4,124 +4,173 @@ 该插件给 EMQ X 带来的扩展性十分的强大，它能以你熟悉语言处理任何的私有协议，并享受由 EMQ X 系统带来的高连接，和高并发的优点。 +**声明：当前仅实现了 Python、Java 的支持** + ## 特性 -- 极强的扩展能力。使用 gRPC 作为 RPC 通信框架，支持各个主流编程语言 +- 多语言支持。快速将接入层的协议实现迁移到 EMQ X 中进行管理 - 高吞吐。连接层以完全的异步非阻塞式 I/O 的方式实现 -- 连接层透明。完全的支持 TCP\TLS UDP\DTLS 类型的连接管理，并对上层提供统一个 API +- 完善的连接层。完全的支持 TCP\TLS UDP\DTLS 类型的连接 - 连接层的管理能力。例如，最大连接数，连接和吞吐的速率限制，IP 黑名单 等 -## 架构 +## 架构 ![Extension-Protocol Arch](images/exproto-arch.jpg) -该插件主要需要处理的内容包括： +该插件需要完成的工作包括三部分： -1. **连接层：** 该部分主要**维持 Socket 的生命周期，和数据的收发**。它的功能要求包括： - - 监听某个端口。当有新的 TCP/UDP 连接到达后，启动一个连接进程，来维持连接的状态。 - - 调用 `OnSocketCreated` 回调。用于通知外部模块**已新建立了一个连接**。 - - 调用 `OnScoektClosed` 回调。用于通知外部模块连接**已关闭**。 - - 调用 `OnReceivedBytes` 回调。用于通知外部模块**该连接新收到的数据包**。 - - 提供 `Send` 接口。供外部模块调用，**用于发送数据包**。 - - 提供 `Close` 接口。供外部模块调用，**用于主动关闭连接**。 +**初始化：** (TODO) +- loaded: +- unload: -2. **协议/会话层：**该部分主要**提供 PUB/SUB 接口**，以实现与 EMQ X Broker 系统的消息互通。包括： +**连接层：** 该部分主要**维持 Socket 的生命周期，和数据的收发**。它的功能要求包括： - - 提供 `Authenticate` 接口。供外部模块调用，用于向集群注册客户端。 - - 提供 `StartTimer` 接口。供外部模块调用，用于为该连接进程启动心跳等定时器。 - - 提供 `Publish` 接口。供外部模块调用，用于发布消息 EMQ X Broker 中。 - - 提供 `Subscribe` 接口。供外部模块调用，用于订阅某主题，以实现从 EMQ X Broker 中接收某些下行消息。 - - 提供 `Unsubscribe` 接口。供外部模块调用，用于取消订阅某主题。 - - 调用 `OnTimerTimeout` 回调。用于处理定时器超时的事件。 - - 调用 `OnReceivedMessages` 回调。用于接收下行消息（在订阅主题成功后，如果主题上有消息，便会回调该方法） +- 监听某个端口。当有新的 TCP/UDP 连接到达后，启动一个连接进程，来维持连接的状态。 +- 调用 `init` 回调。用于通知外部模块**已新建立了一个连接**。 +- 调用 `terminated` 回调。用于通知外部模块连接**已关闭**。 +- 调用 `received` 回调。用于通知外部模块**该连接新收到的数据包**。 +- 提供 `send` 接口。供外部模块调用，**用于发送数据包**。 +- 提供 `close` 接口。供外部模块调用，**用于主动关闭连接**。 + + +**协议/会话层：**该部分主要**提供 PUB/SUB 接口**，以实现与 EMQ X Broker 系统的消息互通。包括： + +- 提供 `register` 接口。供外部模块调用，用于向集群注册客户端。 +- 提供 `publish` 接口。供外部模块调用，用于发布消息 EMQ X Broker 中。 +- 提供 `subscribe` 接口。供外部模块调用，用于订阅某主题，以实现从 EMQ X Broker 中接收某些下行消息。 +- 提供 `unsubscribe` 接口。供外部模块调用，用于取消订阅某主题。 +- 调用 `deliver` 回调。用于接收下行消息（在订阅主题成功后，如果主题上有消息，便会回调该方法） + + +**管理&统计相关：** 该部分主要提供其他**管理&统计相关的接口**。包括： + +- 提供 `Hooks` 类的接口。用于与系统的钩子系统进行交互。 +- 提供 `Metrics` 类的接口。用于统计。 +- 提供 `HTTP or CLI` 管理类接口。 ## 接口设计 -从 gRPC 上的逻辑来说，emqx-exproto 会作为客户端向用户的 `ProtocolHandler` 服务发送回调请求。同时，它也会作为服务端向用户提供 `ConnectionAdapter` 服务，以提供 emqx-exproto 各个接口的访问。如图： +### 连接层接口 -![Extension Protocol gRPC Arch](images/exproto-grpc-arch.jpg) +多语言组件需要向 EMQ X 注册的回调函数： +```erlang +%% Got a new Connection +init(conn(), conninfo()) -> state(). -详情参见：`priv/protos/exproto.proto`，例如接口的定义有： +%% Incoming a data +recevied(conn(), data(), state()) -> state(). -```protobuff -syntax = "proto3"; +%% Socket & Connection process terminated +terminated(conn(), reason(), state()) -> ok. -package emqx.exproto.v1; +-opaue conn() :: pid(). -// The Broker side serivce. It provides a set of APIs to -// handle a protcol access -service ConnectionAdapter { +-type conninfo() :: [ {socktype, tcp | tls | udp | dtls}, + , {peername, {inet:ip_address(), inet:port_number()}}, + , {sockname, {inet:ip_address(), inet:port_number()}}, + , {peercert, nossl | [{cn, string()}, {dn, string()}]} + ]). - // -- socket layer +-type reason() :: string(). - rpc Send(SendBytesRequest) returns (CodeResponse) {}; - - rpc Close(CloseSocketRequest) returns (CodeResponse) {}; - - // -- protocol layer - - rpc Authenticate(AuthenticateRequest) returns (CodeResponse) {}; - - rpc StartTimer(TimerRequest) returns (CodeResponse) {}; - - // -- pub/sub layer - - rpc Publish(PublishRequest) returns (CodeResponse) {}; - - rpc Subscribe(SubscribeRequest) returns (CodeResponse) {}; - - rpc Unsubscribe(UnsubscribeRequest) returns (CodeResponse) {}; -} - -service ConnectionHandler { - - // -- socket layer - - rpc OnSocketCreated(SocketCreatedRequest) returns (EmptySuccess) {}; - - rpc OnSocketClosed(SocketClosedRequest) returns (EmptySuccess) {}; - - rpc OnReceivedBytes(ReceivedBytesRequest) returns (EmptySuccess) {}; - - // -- pub/sub layer - - rpc OnTimerTimeout(TimerTimeoutRequest) returns (EmptySuccess) {}; - - rpc OnReceivedMessages(ReceivedMessagesRequest) returns (EmptySuccess) {}; -} +-type state() :: any(). ``` + +`emqx-exproto` 需要向多语言插件提供的接口： + +``` erlang +%% Send a data to socket +send(conn(), data()) -> ok. + +%% Close the socket +close(conn() ) -> ok. +``` + + +### 协议/会话层接口 + +多语言组件需要向 EMQ X 注册的回调函数： + +```erlang +%% Received a message from a Topic +deliver(conn(), [message()], state()) -> state(). + +-type message() :: [ {id, binary()} + , {qos, integer()} + , {from, binary()} + , {topic, binary()} + , {payload, binary()} + , {timestamp, integer()} + ]. +``` + + +`emqx-exproto` 需要向多语言插件提供的接口： + +``` erlang +%% Reigster the client to Broker +register(conn(), clientinfo()) -> ok | {error, Reason}. + +%% Publish a message to Broker +publish(conn(), message()) -> ok. + +%% Subscribe a topic +subscribe(conn(), topic(), qos()) -> ok. + +%% Unsubscribe a topic +unsubscribe(conn(), topic()) -> ok. + +-type clientinfo() :: [ {proto_name, binary()} + , {proto_ver, integer() | string()} + , {clientid, binary()} + , {username, binary()} + , {mountpoint, binary()}} + , {keepalive, non_neg_integer()} + ]. +``` + +### 管理&统计相关接口 + +*TODO..* + ## 配置项设计 1. 以 **监听器( Listener)** 为基础，提供 TCP/UDP 的监听。 - Listener 目前仅支持：TCP、TLS、UDP、DTLS。(ws、wss、quic 暂不支持) -2. 每个监听器，会指定一个 `ProtocolHandler` 的服务地址，用于调用外部模块的接口。 -3. emqx-exproto 还会监听一个 gRPC 端口用于提供对 `ConnectionAdapter` 服务的访问。 +2. 每个监听器，会指定一个多语言的驱动，用于调用外部模块的接口 + - Driver 目前仅支持：python，java 例如： ``` properties -## gRPC 服务监听地址 (HTTP) -## -exproto.server.http.url = http://127.0.0.1:9002 +## A JT/T 808 TCP based example: +exproto.listener.jtt808 = 6799 +exproto.listener.jtt808.type = tcp +exproto.listener.jtt808.driver = python +# acceptors, max_connections, max_conn_rate, ... +# proxy_protocol, ... +# sndbuff, recbuff, ... +# ssl, cipher, certfile, psk, ... -## gRPC 服务监听地址 (HTTPS) -## -exproto.server.https.url = https://127.0.0.1:9002 -exproto.server.https.cacertfile = ca.pem -exproto.server.https.certfile = cert.pem -exproto.server.https.keyfile = key.pem - -## Listener 配置 -## 例如，名称为 protoname 协议的 TCP 监听器配置 -exproto.listener.protoname = tcp://0.0.0.0:7993 - -## ProtocolHandler 服务地址及 https 的证书配置 -exproto.listener.protoname.proto_handler_url = http://127.0.0.1:9001 -#exproto.listener.protoname.proto_handler_certfile = -#exproto.listener.protoname.proto_handler_cacertfile = -#exproto.listener.protoname.proto_handler_keyfile = +exproto.listener.jtt808. = +## A CoAP UDP based example +exproto.listener.coap = 6799 +exproto.listener.coap.type = udp +exproto.listener.coap.driver = java # ... ``` + +## 集成与调试 + +参见 SDK 规范、和对应语言的开发手册 + +## SDK 实现要求 + +参见 SDK 规范、和对应语言的开发手册 + +## TODOs: + +- 认证 和 发布 订阅鉴权等钩子接入 diff --git a/apps/emqx_exproto/docs/images/exproto-arch.jpg b/apps/emqx_exproto/docs/images/exproto-arch.jpg index dddf7996b..54cd63f61 100644 Binary files a/apps/emqx_exproto/docs/images/exproto-arch.jpg and b/apps/emqx_exproto/docs/images/exproto-arch.jpg differ diff --git a/apps/emqx_exproto/include/emqx_exproto.hrl b/apps/emqx_exproto/include/emqx_exproto.hrl index 079a1e60f..7dcb377f8 100644 --- a/apps/emqx_exproto/include/emqx_exproto.hrl +++ b/apps/emqx_exproto/include/emqx_exproto.hrl @@ -22,16 +22,3 @@ %% TODO: -define(UDP_SOCKOPTS, []). -%%-------------------------------------------------------------------- -%% gRPC result code - --define(RESP_UNKNOWN, 'UNKNOWN'). --define(RESP_SUCCESS, 'SUCCESS'). --define(RESP_CONN_PROCESS_NOT_ALIVE, 'CONN_PROCESS_NOT_ALIVE'). --define(RESP_PARAMS_TYPE_ERROR, 'PARAMS_TYPE_ERROR'). --define(RESP_REQUIRED_PARAMS_MISSED, 'REQUIRED_PARAMS_MISSED'). --define(RESP_PERMISSION_DENY, 'PERMISSION_DENY'). --define(IS_GRPC_RESULT_CODE(C), ( C =:= ?RESP_SUCCESS - orelse C =:= ?RESP_CONN_PROCESS_NOT_ALIVE - orelse C =:= ?RESP_REQUIRED_PARAMS_MISSED - orelse C =:= ?RESP_PERMISSION_DENY)). diff --git a/apps/emqx_exproto/priv/emqx_exproto.schema b/apps/emqx_exproto/priv/emqx_exproto.schema index fb114dc77..af63f56f9 100644 --- a/apps/emqx_exproto/priv/emqx_exproto.schema +++ b/apps/emqx_exproto/priv/emqx_exproto.schema @@ -1,66 +1,25 @@ %% -*-: erlang -*- - -%%-------------------------------------------------------------------- -%% Services - -{mapping, "exproto.server.http.port", "emqx_exproto.servers", [ - {datatype, integer} -]}. - -{mapping, "exproto.server.https.port", "emqx_exproto.servers", [ - {datatype, integer} -]}. - -{mapping, "exproto.server.https.cacertfile", "emqx_exproto.servers", [ - {datatype, string} -]}. - -{mapping, "exproto.server.https.certfile", "emqx_exproto.servers", [ - {datatype, string} -]}. - -{mapping, "exproto.server.https.keyfile", "emqx_exproto.servers", [ - {datatype, string} -]}. - -{translation, "emqx_exproto.servers", fun(Conf) -> - Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end, - Http = case cuttlefish:conf_get("exproto.server.http.port", Conf, undefined) of - undefined -> []; - P1 -> [{http, P1, []}] - end, - Https = case cuttlefish:conf_get("exproto.server.https.port", Conf, undefined) of - undefined -> []; - P2 -> - [{https, P2, - Filter([{ssl, true}, - {certfile, cuttlefish:conf_get("exproto.server.https.certfile", Conf)}, - {keyfile, cuttlefish:conf_get("exproto.server.https.keyfile", Conf)}, - {cacertfile, cuttlefish:conf_get("exproto.server.https.cacertfile", Conf)}])}] - end, - Http ++ Https -end}. - %%-------------------------------------------------------------------- %% Listeners +%%-------------------------------------------------------------------- + +%%-------------------------------------------------------------------- +%% TCP Listeners {mapping, "exproto.listener.$proto", "emqx_exproto.listeners", [ {datatype, string} ]}. -{mapping, "exproto.listener.$proto.connection_handler_url", "emqx_exproto.listeners", [ +{mapping, "exproto.listener.$proto.driver", "emqx_exproto.listeners", [ + {datatype, {enum, [python3, java]}} +]}. + +{mapping, "exproto.listener.$proto.driver_search_path", "emqx_exproto.listeners", [ {datatype, string} ]}. -{mapping, "exproto.listener.$proto.connection_handler_certfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.connection_handler_cacertfile", "emqx_exproto.listeners", [ - {datatype, string} -]}. - -{mapping, "exproto.listener.$proto.connection_handler_keyfile", "emqx_exproto.listeners", [ +{mapping, "exproto.listener.$proto.driver_callback_module", "emqx_exproto.listeners", [ + {default, "main"}, {datatype, string} ]}. @@ -231,23 +190,14 @@ end}. {Rate, Limit} end, - HandlerOpts = fun(Prefix) -> - Opts = - case http_uri:parse(cuttlefish:conf_get(Prefix ++ ".connection_handler_url", Conf)) of - {ok, {http, _, Host, Port, _, _}} -> - [{scheme, http}, {host, Host}, {port, Port}]; - {ok, {https, _, Host, Port, _, _}} -> - [{scheme, https}, {host, Host}, {port, Port}, - {ssl_options, - Filter([{certfile, cuttlefish:conf_get(Prefix ++ ".connection_handler_certfile", Conf)}, - {keyfile, cuttlefish:conf_get(Prefix ++ ".connection_handler_keyfile", Conf)}, - {cacertfile, cuttlefish:conf_get(Prefix ++ ".connection_handler_cacertfile", Conf)} - ])}]; - _ -> - error(invaild_connection_handler_url) - end, - [{handler, Opts}] - end, + DriverOpts = fun(Prefix) -> + [{driver, + Filter([{type, cuttlefish:conf_get(Prefix ++ ".driver", Conf)}, + {path, cuttlefish:conf_get(Prefix ++ ".driver_search_path", Conf)}, + {cbm, Atom(cuttlefish:conf_get(Prefix ++ ".driver_callback_module", Conf))} + ]) + }] + end, ConnOpts = fun(Prefix) -> Filter([{active_n, cuttlefish:conf_get(Prefix ++ ".active_n", Conf, undefined)}, @@ -339,7 +289,7 @@ end}. Listeners = fun(Proto) -> Prefix = string:join(["exproto","listener", Proto], "."), - Opts = HandlerOpts(Prefix) ++ ConnOpts(Prefix) ++ LisOpts(Prefix), + Opts = DriverOpts(Prefix) ++ ConnOpts(Prefix) ++ LisOpts(Prefix), case cuttlefish:conf_get(Prefix, Conf, undefined) of undefined -> []; ListenOn0 -> diff --git a/apps/emqx_exproto/rebar.config b/apps/emqx_exproto/rebar.config index bfc87e92c..52225fea3 100644 --- a/apps/emqx_exproto/rebar.config +++ b/apps/emqx_exproto/rebar.config @@ -1,4 +1,7 @@ %%-*- mode: erlang -*- + +{deps, [{erlport, {git, "https://github.com/emqx/erlport", {tag, "v1.2.2"}}}]}. + {edoc_opts, [{preprocess, true}]}. {erl_opts, [warn_unused_vars, @@ -8,38 +11,12 @@ debug_info, {parse_transform}]}. -{plugins, - [{grpcbox_plugin, {git, "https://github.com/zmstone/grpcbox_plugin", {branch, "master"}}} - ]}. - -{deps, - [{grpcbox, {git, "https://github.com/tsloughter/grpcbox", {branch, "master"}}} - ]}. - -{grpc, - [{type, all}, - {protos, ["priv/protos"]}, - {gpb_opts, [{module_name_prefix, "emqx_"}, - {module_name_suffix, "_pb"}]} - ]}. - -{provider_hooks, - [{pre, [{compile, {grpc, gen}}]}]}. - {xref_checks, [undefined_function_calls, undefined_functions, locals_not_used, deprecated_function_calls, warnings_as_errors, deprecated_functions]}. - -{xref_ignores, [emqx_exproto_pb]}. - {cover_enabled, true}. {cover_opts, [verbose]}. {cover_export_enabled, true}. -{cover_excl_mods, [emqx_exproto_pb, - emqx_exproto_v_1_connection_adapter_client, - emqx_exproto_v_1_connection_adapter_bhvr, - emqx_exproto_v_1_connection_handler_client, - emqx_exproto_v_1_connection_handler_bhvr]}. {profiles, [{test, [ diff --git a/apps/emqx_exproto/src/emqx_exproto.app.src b/apps/emqx_exproto/src/emqx_exproto.app.src index 6ac404f47..2d445570b 100644 --- a/apps/emqx_exproto/src/emqx_exproto.app.src +++ b/apps/emqx_exproto/src/emqx_exproto.app.src @@ -1,12 +1,14 @@ {application, emqx_exproto, [{description, "EMQ X Extension for Protocol"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "4.3.0"}, %% strict semver {modules, []}, {registered, []}, {mod, {emqx_exproto_app, []}}, - {applications, [kernel,stdlib]}, + {applications, [kernel, stdlib, erlport]}, {env,[]}, {licenses, ["Apache-2.0"]}, {maintainers, ["EMQ X Team "]}, - {links, [{"Homepage", "https://emqx.io/"}]} + {links, [{"Homepage", "https://emqx.io/"}, + {"Github", "https://github.com/emqx/emqx-extension-proto"} + ]} ]}. diff --git a/apps/emqx_exproto/src/emqx_exproto.erl b/apps/emqx_exproto/src/emqx_exproto.erl index 8ec382901..c8c36b19e 100644 --- a/apps/emqx_exproto/src/emqx_exproto.erl +++ b/apps/emqx_exproto/src/emqx_exproto.erl @@ -16,20 +16,24 @@ -module(emqx_exproto). +-compile({no_auto_import, [register/1]}). + -include("emqx_exproto.hrl"). -export([ start_listeners/0 , stop_listeners/0 - , start_listener/1 - , start_listener/4 - , stop_listener/4 - , stop_listener/1 ]). --export([ start_servers/0 - , stop_servers/0 - , start_server/1 - , stop_server/1 +%% APIs: Connection level +-export([ send/2 + , close/1 + ]). + +%% APIs: Protocol/Session level +-export([ register/2 + , publish/2 + , subscribe/3 + , unsubscribe/2 ]). %%-------------------------------------------------------------------- @@ -38,71 +42,78 @@ -spec(start_listeners() -> ok). start_listeners() -> - Listeners = application:get_env(?APP, listeners, []), - NListeners = [start_connection_handler_instance(Listener) - || Listener <- Listeners], - lists:foreach(fun start_listener/1, NListeners). + lists:foreach(fun start_listener/1, application:get_env(?APP, listeners, [])). -spec(stop_listeners() -> ok). stop_listeners() -> - Listeners = application:get_env(?APP, listeners, []), - lists:foreach(fun stop_connection_handler_instance/1, Listeners), - lists:foreach(fun stop_listener/1, Listeners). + lists:foreach(fun stop_listener/1, application:get_env(?APP, listeners, [])). --spec(start_servers() -> ok). -start_servers() -> - lists:foreach(fun start_server/1, application:get_env(?APP, servers, [])). +%%-------------------------------------------------------------------- +%% APIs - Connection level +%%-------------------------------------------------------------------- --spec(stop_servers() -> ok). -stop_servers() -> - lists:foreach(fun stop_server/1, application:get_env(?APP, servers, [])). +-spec(send(pid(), binary()) -> ok). +send(Conn, Data) when is_pid(Conn), is_binary(Data) -> + emqx_exproto_conn:cast(Conn, {send, Data}). + +-spec(close(pid()) -> ok). +close(Conn) when is_pid(Conn) -> + emqx_exproto_conn:cast(Conn, close). + +%%-------------------------------------------------------------------- +%% APIs - Protocol/Session level +%%-------------------------------------------------------------------- + +-spec(register(pid(), list()) -> ok | {error, any()}). +register(Conn, ClientInfo0) -> + case emqx_exproto_types:parse(clientinfo, ClientInfo0) of + {error, Reason} -> + {error, Reason}; + ClientInfo -> + emqx_exproto_conn:cast(Conn, {register, ClientInfo}) + end. + +-spec(publish(pid(), list()) -> ok | {error, any()}). +publish(Conn, Msg0) when is_pid(Conn), is_list(Msg0) -> + case emqx_exproto_types:parse(message, Msg0) of + {error, Reason} -> + {error, Reason}; + Msg -> + emqx_exproto_conn:cast(Conn, {publish, Msg}) + end. + +-spec(subscribe(pid(), binary(), emqx_types:qos()) -> ok | {error, any()}). +subscribe(Conn, Topic, Qos) + when is_pid(Conn), is_binary(Topic), + (Qos =:= 0 orelse Qos =:= 1 orelse Qos =:= 2) -> + emqx_exproto_conn:cast(Conn, {subscribe, Topic, Qos}). + +-spec(unsubscribe(pid(), binary()) -> ok | {error, any()}). +unsubscribe(Conn, Topic) + when is_pid(Conn), is_binary(Topic) -> + emqx_exproto_conn:cast(Conn, {unsubscribe, Topic}). %%-------------------------------------------------------------------- %% Internal functions %%-------------------------------------------------------------------- -start_connection_handler_instance({_Proto, _LisType, _ListenOn, Opts}) -> - Name = name(_Proto, _LisType), - {value, {_, HandlerOpts}, LisOpts} = lists:keytake(handler, 1, Opts), - {Endpoints, ChannelOptions} = handler_opts(HandlerOpts), - case emqx_exproto_sup:start_grpc_client_channel(Name, Endpoints, ChannelOptions) of - {ok, _ClientChannelPid} -> - {_Proto, _LisType, _ListenOn, [{handler, Name} | LisOpts]}; - {error, Reason} -> - io:format(standard_error, "Failed to start ~s's connection handler - ~0p~n!", - [Name, Reason]), - error(Reason) - end. - -stop_connection_handler_instance({_Proto, _LisType, _ListenOn, _Opts}) -> - Name = name(_Proto, _LisType), - _ = emqx_exproto_sup:stop_grpc_client_channel(Name), - ok. - -start_server({Name, Port, SSLOptions}) -> - case emqx_exproto_sup:start_grpc_server(Name, Port, SSLOptions) of - {ok, _} -> - io:format("Start ~s gRPC server on ~w successfully.~n", - [Name, Port]); - {error, Reason} -> - io:format(standard_error, "Failed to start ~s gRPC server on ~w - ~0p~n!", - [Name, Port, Reason]), - error({failed_start_server, Reason}) - end. - -stop_server({Name, Port, _SSLOptions}) -> - ok = emqx_exproto_sup:stop_grpc_server(Name), - io:format("Stop ~s gRPC server on ~w successfully.~n", [Name, Port]). - start_listener({Proto, LisType, ListenOn, Opts}) -> Name = name(Proto, LisType), - case start_listener(LisType, Name, ListenOn, Opts) of - {ok, _} -> - io:format("Start ~s listener on ~s successfully.~n", - [Name, format(ListenOn)]); + {value, {_, DriverOpts}, LisOpts} = lists:keytake(driver, 1, Opts), + case emqx_exproto_driver_mngr:ensure_driver(Name, DriverOpts) of + {ok, _DriverPid}-> + case start_listener(LisType, Name, ListenOn, [{driver, Name} |LisOpts]) of + {ok, _} -> + io:format("Start ~s listener on ~s successfully.~n", + [Name, format(ListenOn)]); + {error, Reason} -> + io:format(standard_error, "Failed to start ~s listener on ~s - ~0p~n!", + [Name, format(ListenOn), Reason]), + error(Reason) + end; {error, Reason} -> - io:format(standard_error, "Failed to start ~s listener on ~s - ~0p~n!", - [Name, format(ListenOn), Reason]), + io:format(standard_error, "Failed to start ~s's driver - ~0p~n!", + [Name, Reason]), error(Reason) end. @@ -126,11 +137,11 @@ start_listener(dtls, Name, ListenOn, LisOpts) -> stop_listener({Proto, LisType, ListenOn, Opts}) -> Name = name(Proto, LisType), + _ = emqx_exproto_driver_mngr:stop_driver(Name), StopRet = stop_listener(LisType, Name, ListenOn, Opts), case StopRet of - ok -> - io:format("Stop ~s listener on ~s successfully.~n", - [Name, format(ListenOn)]); + ok -> io:format("Stop ~s listener on ~s successfully.~n", + [Name, format(ListenOn)]); {error, Reason} -> io:format(standard_error, "Failed to stop ~s listener on ~s - ~p~n.", [Name, format(ListenOn), Reason]) @@ -146,12 +157,8 @@ name(Proto, LisType) -> list_to_atom(lists:flatten(io_lib:format("~s:~s", [Proto, LisType]))). %% @private -format(Port) when is_integer(Port) -> - io_lib:format("0.0.0.0:~w", [Port]); format({Addr, Port}) when is_list(Addr) -> - io_lib:format("~s:~w", [Addr, Port]); -format({Addr, Port}) when is_tuple(Addr) -> - io_lib:format("~s:~w", [inet:ntoa(Addr), Port]). + io_lib:format("~s:~w", [Addr, Port]). %% @private merge_tcp_default(Opts) -> @@ -169,15 +176,3 @@ merge_udp_default(Opts) -> false -> [{udp_options, ?UDP_SOCKOPTS} | Opts] end. - -%% @private -handler_opts(Opts) -> - Scheme = proplists:get_value(scheme, Opts), - Host = proplists:get_value(host, Opts), - Port = proplists:get_value(port, Opts), - Options = proplists:get_value(options, Opts, []), - SslOpts = case Scheme of - https -> proplists:get_value(ssl_options, Opts, []); - _ -> [] - end, - {[{Scheme, Host, Port, SslOpts}], maps:from_list(Options)}. diff --git a/apps/emqx_exproto/src/emqx_exproto_app.erl b/apps/emqx_exproto/src/emqx_exproto_app.erl index 73e8a65bc..b12101fda 100644 --- a/apps/emqx_exproto/src/emqx_exproto_app.erl +++ b/apps/emqx_exproto/src/emqx_exproto_app.erl @@ -24,14 +24,13 @@ start(_StartType, _StartArgs) -> {ok, Sup} = emqx_exproto_sup:start_link(), - emqx_exproto:start_servers(), emqx_exproto:start_listeners(), {ok, Sup}. prep_stop(State) -> - emqx_exproto:stop_servers(), emqx_exproto:stop_listeners(), State. stop(_State) -> ok. + diff --git a/apps/emqx_exproto/src/emqx_exproto_channel.erl b/apps/emqx_exproto/src/emqx_exproto_channel.erl index 9786c12c2..5a8e2c888 100644 --- a/apps/emqx_exproto/src/emqx_exproto_channel.erl +++ b/apps/emqx_exproto/src/emqx_exproto_channel.erl @@ -16,7 +16,6 @@ -module(emqx_exproto_channel). --include("emqx_exproto.hrl"). -include_lib("emqx/include/emqx.hrl"). -include_lib("emqx/include/emqx_mqtt.hrl"). -include_lib("emqx/include/types.hrl"). @@ -42,26 +41,20 @@ -export_type([channel/0]). -record(channel, { - %% gRPC channel options - gcli :: map(), + %% Driver name + driver :: atom(), %% Conn info conninfo :: emqx_types:conninfo(), %% Client info from `register` function clientinfo :: maybe(map()), + %% Registered + registered = false :: boolean(), %% Connection state conn_state :: conn_state(), %% Subscription subscriptions = #{}, - %% Request queue - rqueue = queue:new(), - %% Inflight function name - inflight = undefined, - %% Keepalive - keepalive :: maybe(emqx_keepalive:keepalive()), - %% Timers - timers :: #{atom() => disabled | maybe(reference())}, - %% Closed reason - closed_reason = undefined + %% Driver level state + state :: any() }). -opaque(channel() :: #channel{}). @@ -74,11 +67,6 @@ -type(replies() :: emqx_types:packet() | reply() | [reply()]). --define(TIMER_TABLE, #{ - alive_timer => keepalive, - force_timer => force_close - }). - -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session, will_msg]). -define(SESSION_STATS_KEYS, @@ -142,44 +130,20 @@ stats(#channel{subscriptions = Subs}) -> %%-------------------------------------------------------------------- -spec(init(emqx_exproto_types:conninfo(), proplists:proplist()) -> channel()). -init(ConnInfo = #{socktype := Socktype, - peername := Peername, - sockname := Sockname, - peercert := Peercert}, Options) -> - GRpcChann = proplists:get_value(handler, Options), - NConnInfo = default_conninfo(ConnInfo), - ClientInfo = default_clientinfo(ConnInfo), - Channel = #channel{gcli = #{channel => GRpcChann}, - conninfo = NConnInfo, - clientinfo = ClientInfo, - conn_state = connecting, - timers = #{} - }, - - Req = #{conninfo => - peercert(Peercert, - #{socktype => socktype(Socktype), - peername => address(Peername), - sockname => address(Sockname)})}, - try_dispatch(on_socket_created, wrap(Req), Channel). - -%% @private -peercert(nossl, ConnInfo) -> - ConnInfo; -peercert(Peercert, ConnInfo) -> - ConnInfo#{peercert => - #{cn => esockd_peercert:common_name(Peercert), - dn => esockd_peercert:subject(Peercert)}}. - -%% @private -socktype(tcp) -> 'TCP'; -socktype(ssl) -> 'SSL'; -socktype(udp) -> 'UDP'; -socktype(dtls) -> 'DTLS'. - -%% @private -address({Host, Port}) -> - #{host => inet:ntoa(Host), port => Port}. +init(ConnInfo, Options) -> + Driver = proplists:get_value(driver, Options), + case cb_init(ConnInfo, Driver) of + {ok, DState} -> + NConnInfo = default_conninfo(ConnInfo), + ClientInfo = default_clientinfo(ConnInfo), + #channel{driver = Driver, + state = DState, + conninfo = NConnInfo, + clientinfo = ClientInfo, + conn_state = connected}; + {error, Reason} -> + exit({init_channel_failed, Reason}) + end. %%-------------------------------------------------------------------- %% Handle incoming packet @@ -189,163 +153,81 @@ address({Host, Port}) -> -> {ok, channel()} | {shutdown, Reason :: term(), channel()}). handle_in(Data, Channel) -> - Req = #{bytes => Data}, - {ok, try_dispatch(on_received_bytes, wrap(Req), Channel)}. + case cb_received(Data, Channel) of + {ok, NChannel} -> + {ok, NChannel}; + {error, Reason} -> + {shutdown, Reason, Channel} + end. -spec(handle_deliver(list(emqx_types:deliver()), channel()) -> {ok, channel()} | {shutdown, Reason :: term(), channel()}). -handle_deliver(Delivers, Channel = #channel{clientinfo = ClientInfo}) -> - %% XXX: ?? Nack delivers from shared subscriptions - Mountpoint = maps:get(mountpoint, ClientInfo), - NodeStr = atom_to_binary(node(), utf8), - Msgs = lists:map(fun({_, _, Msg}) -> - ok = emqx_metrics:inc('messages.delivered'), - Msg1 = emqx_hooks:run_fold('message.delivered', - [ClientInfo], Msg), - NMsg = emqx_mountpoint:unmount(Mountpoint, Msg1), - #{node => NodeStr, - id => hexstr(emqx_message:id(NMsg)), - qos => emqx_message:qos(NMsg), - from => fmt_from(emqx_message:from(NMsg)), - topic => emqx_message:topic(NMsg), - payload => emqx_message:payload(NMsg), - timestamp => emqx_message:timestamp(NMsg) - } - end, Delivers), - Req = #{messages => Msgs}, - {ok, try_dispatch(on_received_messages, wrap(Req), Channel)}. +handle_deliver(Delivers, Channel) -> + %% TODO: ?? Nack delivers from shared subscriptions + case cb_deliver(Delivers, Channel) of + {ok, NChannel} -> + {ok, NChannel}; + {error, Reason} -> + {shutdown, Reason, Channel} + end. -spec(handle_timeout(reference(), Msg :: term(), channel()) -> {ok, channel()} | {shutdown, Reason :: term(), channel()}). -handle_timeout(_TRef, {keepalive, _StatVal}, - Channel = #channel{keepalive = undefined}) -> - {ok, Channel}; -handle_timeout(_TRef, {keepalive, StatVal}, - Channel = #channel{keepalive = Keepalive}) -> - case emqx_keepalive:check(StatVal, Keepalive) of - {ok, NKeepalive} -> - NChannel = Channel#channel{keepalive = NKeepalive}, - {ok, reset_timer(alive_timer, NChannel)}; - {error, timeout} -> - Req = #{type => 'KEEPALIVE'}, - {ok, try_dispatch(on_timer_timeout, wrap(Req), Channel)} - end; - -handle_timeout(_TRef, force_close, Channel = #channel{closed_reason = Reason}) -> - {shutdown, {error, {force_close, Reason}}, Channel}; - handle_timeout(_TRef, Msg, Channel) -> ?WARN("Unexpected timeout: ~p", [Msg]), {ok, Channel}. -spec(handle_call(any(), channel()) -> {reply, Reply :: term(), channel()} - | {reply, Reply :: term(), replies(), channel()} | {shutdown, Reason :: term(), Reply :: term(), channel()}). - -handle_call({send, Data}, Channel) -> - {reply, ok, [{outgoing, Data}], Channel}; - -handle_call(close, Channel = #channel{conn_state = connected}) -> - {reply, ok, [{event, disconnected}, {close, normal}], Channel}; -handle_call(close, Channel) -> - {reply, ok, [{close, normal}], Channel}; - -handle_call({auth, ClientInfo, _Password}, Channel = #channel{conn_state = connected}) -> - ?LOG(warning, "Duplicated authorized command, dropped ~p", [ClientInfo]), - {ok, {error, ?RESP_PERMISSION_DENY, <<"Duplicated authenticate command">>}, Channel}; -handle_call({auth, ClientInfo0, Password}, - Channel = #channel{conninfo = ConnInfo, - clientinfo = ClientInfo}) -> - ClientInfo1 = enrich_clientinfo(ClientInfo0, ClientInfo), - NConnInfo = enrich_conninfo(ClientInfo1, ConnInfo), - - Channel1 = Channel#channel{conninfo = NConnInfo, - clientinfo = ClientInfo1}, - - #{clientid := ClientId, username := Username} = ClientInfo1, - - case emqx_access_control:authenticate(ClientInfo1#{password => Password}) of - {ok, AuthResult} -> - emqx_logger:set_metadata_clientid(ClientId), - is_anonymous(AuthResult) andalso - emqx_metrics:inc('client.auth.anonymous'), - NClientInfo = maps:merge(ClientInfo1, AuthResult), - NChannel = Channel1#channel{clientinfo = NClientInfo}, - case emqx_cm:open_session(true, NClientInfo, NConnInfo) of - {ok, _Session} -> - ?LOG(debug, "Client ~s (Username: '~s') authorized successfully!", - [ClientId, Username]), - {reply, ok, [{event, connected}], ensure_connected(NChannel)}; - {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') open session failed for ~0p", - [ClientId, Username, Reason]), - {reply, {error, ?RESP_PERMISSION_DENY, Reason}, Channel} - end; - {error, Reason} -> - ?LOG(warning, "Client ~s (Username: '~s') login failed for ~0p", - [ClientId, Username, Reason]), - {reply, {error, ?RESP_PERMISSION_DENY, Reason}, Channel} - end; - -handle_call({start_timer, keepalive, Interval}, - Channel = #channel{ - conninfo = ConnInfo, - clientinfo = ClientInfo - }) -> - NConnInfo = ConnInfo#{keepalive => Interval}, - NClientInfo = ClientInfo#{keepalive => Interval}, - NChannel = Channel#channel{conninfo = NConnInfo, clientinfo = NClientInfo}, - {reply, ok, ensure_keepalive(NChannel)}; - -handle_call({subscribe, TopicFilter, Qos}, - Channel = #channel{ - conn_state = connected, - clientinfo = ClientInfo}) -> - case is_acl_enabled(ClientInfo) andalso - emqx_access_control:check_acl(ClientInfo, subscribe, TopicFilter) of - deny -> - {reply, {error, ?RESP_PERMISSION_DENY, <<"ACL deny">>}, Channel}; - _ -> - {ok, NChannel} = do_subscribe([{TopicFilter, #{qos => Qos}}], Channel), - {reply, ok, NChannel} - end; - -handle_call({unsubscribe, TopicFilter}, - Channel = #channel{conn_state = connected}) -> - {ok, NChannel} = do_unsubscribe([{TopicFilter, #{}}], Channel), - {reply, ok, NChannel}; - -handle_call({publish, Topic, Qos, Payload}, - Channel = #channel{ - conn_state = connected, - clientinfo = ClientInfo - = #{clientid := From, - mountpoint := Mountpoint}}) -> - case is_acl_enabled(ClientInfo) andalso - emqx_access_control:check_acl(ClientInfo, publish, Topic) of - deny -> - {reply, {error, ?RESP_PERMISSION_DENY, <<"ACL deny">>}, Channel}; - _ -> - Msg = emqx_message:make(From, Qos, Topic, Payload), - NMsg = emqx_mountpoint:mount(Mountpoint, Msg), - emqx:publish(NMsg), - {reply, ok, Channel} - end; - handle_call(kick, Channel) -> {shutdown, kicked, ok, Channel}; handle_call(Req, Channel) -> - ?LOG(warning, "Unexpected call: ~p", [Req]), - {reply, {error, unexpected_call}, Channel}. + ?WARN("Unexpected call: ~p", [Req]), + {reply, ok, Channel}. -spec(handle_cast(any(), channel()) -> {ok, channel()} | {ok, replies(), channel()} | {shutdown, Reason :: term(), channel()}). +handle_cast({send, Data}, Channel) -> + {ok, [{outgoing, Data}], Channel}; + +handle_cast(close, Channel) -> + {ok, [{close, normal}], Channel}; + +handle_cast({register, ClientInfo}, Channel = #channel{registered = true}) -> + ?WARN("Duplicated register command, dropped ~p", [ClientInfo]), + {ok, Channel}; +handle_cast({register, ClientInfo0}, Channel = #channel{conninfo = ConnInfo, + clientinfo = ClientInfo}) -> + ClientInfo1 = maybe_assign_clientid(ClientInfo0), + NConnInfo = enrich_conninfo(ClientInfo1, ConnInfo), + NClientInfo = enrich_clientinfo(ClientInfo1, ClientInfo), + case emqx_cm:open_session(true, NClientInfo, NConnInfo) of + {ok, _Session} -> + NChannel = Channel#channel{registered = true, + conninfo = NConnInfo, + clientinfo = NClientInfo}, + {ok, [{event, registered}], NChannel}; + {error, Reason} -> + ?ERROR("Register failed, reason: ~p", [Reason]), + {shutdown, Reason, {error, Reason}, Channel} + end; + +handle_cast({subscribe, TopicFilter, Qos}, Channel) -> + do_subscribe([{TopicFilter, #{qos => Qos}}], Channel); + +handle_cast({unsubscribe, TopicFilter}, Channel) -> + do_unsubscribe([{TopicFilter, #{}}], Channel); + +handle_cast({publish, Msg}, Channel) -> + emqx:publish(enrich_msg(Msg, Channel)), + {ok, Channel}; + handle_cast(Req, Channel) -> ?WARN("Unexpected call: ~p", [Req]), {ok, Channel}. @@ -359,41 +241,15 @@ handle_info({subscribe, TopicFilters}, Channel) -> handle_info({unsubscribe, TopicFilters}, Channel) -> do_unsubscribe(TopicFilters, Channel); -handle_info({sock_closed, Reason}, - Channel = #channel{rqueue = Queue, inflight = Inflight}) -> - case queue:len(Queue) =:= 0 - andalso Inflight =:= undefined of - true -> - {shutdown, {sock_closed, Reason}, Channel}; - _ -> - %% delayed close process for flushing all callback funcs to gRPC server - Channel1 = Channel#channel{closed_reason = {sock_closed, Reason}}, - Channel2 = ensure_timer(force_timer, Channel1), - {ok, ensure_disconnected({sock_closed, Reason}, Channel2)} - end; - -handle_info({hreply, on_socket_created, {ok, _}}, Channel) -> - dispatch_or_close_process(Channel#channel{inflight = undefined}); -handle_info({hreply, FunName, {ok, _}}, Channel) - when FunName == on_socket_closed; - FunName == on_received_bytes; - FunName == on_received_messages; - FunName == on_timer_timeout -> - dispatch_or_close_process(Channel#channel{inflight = undefined}); -handle_info({hreply, FunName, {error, Reason}}, Channel) -> - {shutdown, {error, {FunName, Reason}}, Channel}; - +handle_info({sock_closed, Reason}, Channel) -> + {shutdown, {sock_closed, Reason}, Channel}; handle_info(Info, Channel) -> - ?LOG(warning, "Unexpected info: ~p", [Info]), + ?WARN("Unexpected info: ~p", [Info]), {ok, Channel}. --spec(terminate(any(), channel()) -> channel()). +-spec(terminate(any(), channel()) -> ok). terminate(Reason, Channel) -> - Req = #{reason => stringfy(Reason)}, - try_dispatch(on_socket_closed, wrap(Req), Channel). - -is_anonymous(#{anonymous := true}) -> true; -is_anonymous(_AuthResult) -> false. + cb_terminated(Reason, Channel), ok. %%-------------------------------------------------------------------- %% Sub/UnSub @@ -410,22 +266,11 @@ do_subscribe(TopicFilters, Channel) -> do_subscribe(TopicFilter, SubOpts, Channel = #channel{clientinfo = ClientInfo = #{mountpoint := Mountpoint}, subscriptions = Subs}) -> - %% Mountpoint first NTopicFilter = emqx_mountpoint:mount(Mountpoint, TopicFilter), NSubOpts = maps:merge(?DEFAULT_SUBOPTS, SubOpts), SubId = maps:get(clientid, ClientInfo, undefined), - IsNew = not maps:is_key(NTopicFilter, Subs), - case IsNew of - true -> - ok = emqx:subscribe(NTopicFilter, SubId, NSubOpts), - ok = emqx_hooks:run('session.subscribed', - [ClientInfo, NTopicFilter, NSubOpts#{is_new => IsNew}]), - Channel#channel{subscriptions = Subs#{NTopicFilter => NSubOpts}}; - _ -> - %% Update subopts - ok = emqx:subscribe(NTopicFilter, SubId, NSubOpts), - Channel#channel{subscriptions = Subs#{NTopicFilter => NSubOpts}} - end. + _ = emqx:subscribe(NTopicFilter, SubId, NSubOpts), + Channel#channel{subscriptions = Subs#{NTopicFilter => SubOpts}}. do_unsubscribe(TopicFilters, Channel) -> NChannel = lists:foldl( @@ -435,133 +280,74 @@ do_unsubscribe(TopicFilters, Channel) -> {ok, NChannel}. %% @private -do_unsubscribe(TopicFilter, UnSubOpts, Channel = - #channel{clientinfo = ClientInfo = #{mountpoint := Mountpoint}, +do_unsubscribe(TopicFilter, _SubOpts, Channel = + #channel{clientinfo = #{mountpoint := Mountpoint}, subscriptions = Subs}) -> - NTopicFilter = emqx_mountpoint:mount(Mountpoint, TopicFilter), - case maps:find(NTopicFilter, Subs) of - {ok, SubOpts} -> - ok = emqx:unsubscribe(NTopicFilter), - ok = emqx_hooks:run('session.unsubscribed', - [ClientInfo, TopicFilter, maps:merge(SubOpts, UnSubOpts)]), - Channel#channel{subscriptions = maps:remove(NTopicFilter, Subs)}; - _ -> - Channel - end. + TopicFilter1 = emqx_mountpoint:mount(Mountpoint, TopicFilter), + _ = emqx:unsubscribe(TopicFilter1), + Channel#channel{subscriptions = maps:remove(TopicFilter1, Subs)}. %% @private parse_topic_filters(TopicFilters) -> lists:map(fun emqx_topic:parse/1, TopicFilters). --compile({inline, [is_acl_enabled/1]}). -is_acl_enabled(#{zone := Zone, is_superuser := IsSuperuser}) -> - (not IsSuperuser) andalso emqx_zone:enable_acl(Zone). - %%-------------------------------------------------------------------- -%% Ensure & Hooks +%% Cbs for driver %%-------------------------------------------------------------------- -ensure_connected(Channel = #channel{conninfo = ConnInfo, - clientinfo = ClientInfo}) -> - NConnInfo = ConnInfo#{connected_at => erlang:system_time(millisecond)}, - ok = run_hooks('client.connected', [ClientInfo, NConnInfo]), - Channel#channel{conninfo = NConnInfo, - conn_state = connected - }. +cb_init(ConnInfo, Driver) -> + Args = [self(), emqx_exproto_types:serialize(conninfo, ConnInfo)], + emqx_exproto_driver_mngr:call(Driver, {'init', Args}). -ensure_disconnected(Reason, Channel = #channel{ - conn_state = connected, - conninfo = ConnInfo, - clientinfo = ClientInfo}) -> - NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, - ok = run_hooks('client.disconnected', [ClientInfo, Reason, NConnInfo]), - Channel#channel{conninfo = NConnInfo, conn_state = disconnected}; +cb_received(Data, Channel = #channel{state = DState}) -> + Args = [self(), Data, DState], + do_call_cb('received', Args, Channel). -ensure_disconnected(_Reason, Channel = #channel{conninfo = ConnInfo}) -> - NConnInfo = ConnInfo#{disconnected_at => erlang:system_time(millisecond)}, - Channel#channel{conninfo = NConnInfo, conn_state = disconnected}. +cb_terminated(Reason, Channel = #channel{state = DState}) -> + Args = [self(), stringfy(Reason), DState], + do_call_cb('terminated', Args, Channel). -run_hooks(Name, Args) -> - ok = emqx_metrics:inc(Name), emqx_hooks:run(Name, Args). +cb_deliver(Delivers, Channel = #channel{state = DState}) -> + Msgs = [emqx_exproto_types:serialize(message, Msg) || {_, _, Msg} <- Delivers], + Args = [self(), Msgs, DState], + do_call_cb('deliver', Args, Channel). -%%-------------------------------------------------------------------- -%% Enrich Keepalive - -ensure_keepalive(Channel = #channel{clientinfo = ClientInfo}) -> - ensure_keepalive_timer(maps:get(keepalive, ClientInfo, 0), Channel). - -ensure_keepalive_timer(Interval, Channel) when Interval =< 0 -> - Channel; -ensure_keepalive_timer(Interval, Channel) -> - Keepalive = emqx_keepalive:init(timer:seconds(Interval)), - ensure_timer(alive_timer, Channel#channel{keepalive = Keepalive}). - -ensure_timer(Name, Channel = #channel{timers = Timers}) -> - TRef = maps:get(Name, Timers, undefined), - Time = interval(Name, Channel), - case TRef == undefined andalso Time > 0 of - true -> ensure_timer(Name, Time, Channel); - false -> Channel %% Timer disabled or exists +%% @private +do_call_cb(Fun, Args, Channel = #channel{driver = D}) -> + case emqx_exproto_driver_mngr:call(D, {Fun, Args}) of + ok -> + {ok, Channel}; + {ok, NDState} -> + {ok, Channel#channel{state = NDState}}; + {error, Reason} -> + {error, Reason} end. -ensure_timer(Name, Time, Channel = #channel{timers = Timers}) -> - Msg = maps:get(Name, ?TIMER_TABLE), - TRef = emqx_misc:start_timer(Time, Msg), - Channel#channel{timers = Timers#{Name => TRef}}. - -reset_timer(Name, Channel) -> - ensure_timer(Name, clean_timer(Name, Channel)). - -clean_timer(Name, Channel = #channel{timers = Timers}) -> - Channel#channel{timers = maps:remove(Name, Timers)}. - -interval(force_timer, _) -> - 15000; -interval(alive_timer, #channel{keepalive = Keepalive}) -> - emqx_keepalive:info(interval, Keepalive). - -%%-------------------------------------------------------------------- -%% Dispatch -%%-------------------------------------------------------------------- - -wrap(Req) -> - Req#{conn => pid_to_list(self())}. - -dispatch_or_close_process(Channel = #channel{ - rqueue = Queue, - inflight = undefined, - gcli = GClient}) -> - case queue:out(Queue) of - {empty, _} -> - case Channel#channel.conn_state of - disconnected -> - {shutdown, Channel#channel.closed_reason, Channel}; - _ -> - {ok, Channel} - end; - {{value, {FunName, Req}}, NQueue} -> - emqx_exproto_gcli:async_call(FunName, Req, GClient), - {ok, Channel#channel{inflight = FunName, rqueue = NQueue}} - end. - -try_dispatch(FunName, Req, Channel = #channel{inflight = undefined, gcli = GClient}) -> - emqx_exproto_gcli:async_call(FunName, Req, GClient), - Channel#channel{inflight = FunName}; -try_dispatch(FunName, Req, Channel = #channel{rqueue = Queue}) -> - Channel#channel{rqueue = queue:in({FunName, Req}, Queue)}. - %%-------------------------------------------------------------------- %% Format %%-------------------------------------------------------------------- +maybe_assign_clientid(ClientInfo) -> + case maps:get(clientid, ClientInfo, undefined) of + undefined -> + ClientInfo#{clientid => emqx_guid:to_base62(emqx_guid:gen())}; + _ -> + ClientInfo + end. + +enrich_msg(Msg, #channel{clientinfo = ClientInfo = #{mountpoint := Mountpoint}}) -> + NMsg = emqx_mountpoint:mount(Mountpoint, Msg), + case maps:get(clientid, ClientInfo, undefined) of + undefined -> NMsg; + ClientId -> NMsg#message{from = ClientId} + end. + enrich_conninfo(InClientInfo, ConnInfo) -> - Ks = [proto_name, proto_ver, clientid, username], - maps:merge(ConnInfo, maps:with(Ks, InClientInfo)). + maps:merge(ConnInfo, maps:with([proto_name, proto_ver, clientid, username, keepalive], InClientInfo)). enrich_clientinfo(InClientInfo = #{proto_name := ProtoName}, ClientInfo) -> - Ks = [clientid, username, mountpoint], - NClientInfo = maps:merge(ClientInfo, maps:with(Ks, InClientInfo)), - NClientInfo#{protocol => ProtoName}. + NClientInfo = maps:merge(ClientInfo, maps:with([clientid, username, mountpoint], InClientInfo)), + NClientInfo#{protocol => lowcase_atom(ProtoName)}. default_conninfo(ConnInfo) -> ConnInfo#{proto_name => undefined, @@ -577,12 +363,12 @@ default_conninfo(ConnInfo) -> expiry_interval => 0}. default_clientinfo(#{peername := {PeerHost, _}, - sockname := {_, SockPort}}) -> - #{zone => external, + sockname := {_, SockPort}}) -> + #{zone => undefined, protocol => undefined, peerhost => PeerHost, sockport => SockPort, - clientid => undefined, + clientid => default_clientid(), username => undefined, is_bridge => false, is_superuser => false, @@ -591,9 +377,10 @@ default_clientinfo(#{peername := {PeerHost, _}, stringfy(Reason) -> unicode:characters_to_binary((io_lib:format("~0p", [Reason]))). -hexstr(Bin) -> - [io_lib:format("~2.16.0B",[X]) || <> <= Bin]. +lowcase_atom(undefined) -> + undefined; +lowcase_atom(S) -> + binary_to_atom(string:lowercase(S), utf8). -fmt_from(undefined) -> <<>>; -fmt_from(Bin) when is_binary(Bin) -> Bin; -fmt_from(T) -> stringfy(T). +default_clientid() -> + <<"exproto_client_", (list_to_binary(pid_to_list(self())))/binary>>. diff --git a/apps/emqx_exproto/src/emqx_exproto_conn.erl b/apps/emqx_exproto/src/emqx_exproto_conn.erl index d3ce75f0b..10a40c987 100644 --- a/apps/emqx_exproto/src/emqx_exproto_conn.erl +++ b/apps/emqx_exproto/src/emqx_exproto_conn.erl @@ -173,10 +173,8 @@ esockd_wait({esockd_transport, Sock}) -> R = {error, _} -> R end. -esockd_close({udp, _SockPid, _Sock}) -> - %% nothing to do for udp socket - %%gen_udp:close(Sock); - ok; +esockd_close({udp, _SockPid, Sock}) -> + gen_udp:close(Sock); esockd_close({esockd_transport, Sock}) -> esockd_transport:fast_close(Sock). @@ -359,9 +357,6 @@ handle_msg({'$gen_call', From, Req}, State) -> {reply, Reply, NState} -> gen_server:reply(From, Reply), {ok, NState}; - {reply, Reply, Msgs, NState} -> - gen_server:reply(From, Reply), - {ok, next_msgs(Msgs), NState}; {stop, Reason, Reply, NState} -> gen_server:reply(From, Reply), stop(Reason, NState) @@ -424,16 +419,16 @@ handle_msg({close, Reason}, State) -> ?LOG(debug, "Force to close the socket due to ~p", [Reason]), handle_info({sock_closed, Reason}, close_socket(State)); -handle_msg({event, connected}, State = #state{channel = Channel}) -> +handle_msg({event, registered}, State = #state{channel = Channel}) -> ClientId = emqx_exproto_channel:info(clientid, Channel), emqx_cm:register_channel(ClientId, info(State), stats(State)); -handle_msg({event, disconnected}, State = #state{channel = Channel}) -> - ClientId = emqx_exproto_channel:info(clientid, Channel), - emqx_cm:set_chan_info(ClientId, info(State)), - emqx_cm:connection_closed(ClientId), - {ok, State}; - +%handle_msg({event, disconnected}, State = #state{channel = Channel}) -> +% ClientId = emqx_exproto_channel:info(clientid, Channel), +% emqx_cm:set_chan_info(ClientId, info(State)), +% emqx_cm:connection_closed(ClientId), +% {ok, State}; +% %handle_msg({event, _Other}, State = #state{channel = Channel}) -> % ClientId = emqx_exproto_channel:info(clientid, Channel), % emqx_cm:set_chan_info(ClientId, info(State)), @@ -485,8 +480,6 @@ handle_call(_From, Req, State = #state{channel = Channel}) -> case emqx_exproto_channel:handle_call(Req, Channel) of {reply, Reply, NChannel} -> {reply, Reply, State#state{channel = NChannel}}; - {reply, Reply, Replies, NChannel} -> - {reply, Reply, Replies, State#state{channel = NChannel}}; {shutdown, Reason, Reply, NChannel} -> shutdown(Reason, Reply, State#state{channel = NChannel}) end. @@ -502,18 +495,7 @@ handle_timeout(_TRef, limit_timeout, State) -> limit_timer = undefined }, handle_info(activate_socket, NState); -handle_timeout(TRef, keepalive, State = #state{socket = Socket, - channel = Channel})-> - case emqx_exproto_channel:info(conn_state, Channel) of - disconnected -> {ok, State}; - _ -> - case esockd_getstat(Socket, [recv_oct]) of - {ok, [{recv_oct, RecvOct}]} -> - handle_timeout(TRef, {keepalive, RecvOct}, State); - {error, Reason} -> - handle_info({sock_error, Reason}, State) - end - end; + handle_timeout(_TRef, emit_stats, State = #state{channel = Channel}) -> ClientId = emqx_exproto_channel:info(clientid, Channel), @@ -683,3 +665,4 @@ stop(Reason, State) -> stop(Reason, Reply, State) -> {stop, Reason, Reply, State}. + diff --git a/apps/emqx_exproto/src/emqx_exproto_gcli.erl b/apps/emqx_exproto/src/emqx_exproto_gcli.erl deleted file mode 100644 index 41af21fb8..000000000 --- a/apps/emqx_exproto/src/emqx_exproto_gcli.erl +++ /dev/null @@ -1,110 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% the gRPC client worker for ConnectionHandler service --module(emqx_exproto_gcli). - --behaviour(gen_server). - --include_lib("emqx/include/logger.hrl"). - --logger_header("[ExProto gClient]"). - -%% APIs --export([async_call/3]). - --export([start_link/2]). - -%% gen_server callbacks --export([ init/1 - , handle_call/3 - , handle_cast/2 - , handle_info/2 - , terminate/2 - , code_change/3 - ]). - --define(CONN_ADAPTER_MOD, emqx_exproto_v_1_connection_handler_client). - -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - -start_link(Pool, Id) -> - gen_server:start_link({local, emqx_misc:proc_name(?MODULE, Id)}, - ?MODULE, [Pool, Id], []). - -async_call(FunName, Req = #{conn := Conn}, Options) -> - cast(pick(Conn), {rpc, FunName, Req, Options, self()}). - -%%-------------------------------------------------------------------- -%% cast, pick -%%-------------------------------------------------------------------- - --compile({inline, [cast/2, pick/1]}). - -cast(Deliver, Msg) -> - gen_server:cast(Deliver, Msg). - -pick(Conn) -> - gproc_pool:pick_worker(exproto_gcli_pool, Conn). - -%%-------------------------------------------------------------------- -%% gen_server callbacks -%%-------------------------------------------------------------------- - -init([Pool, Id]) -> - true = gproc_pool:connect_worker(Pool, {Pool, Id}), - {ok, #{pool => Pool, id => Id}}. - -handle_call(_Request, _From, State) -> - {reply, ok, State}. - -handle_cast({rpc, Fun, Req, Options, From}, State) -> - case catch apply(?CONN_ADAPTER_MOD, Fun, [Req, Options]) of - {ok, Resp, _Metadata} -> - ?LOG(debug, "~p got {ok, ~0p, ~0p}", [Fun, Resp, _Metadata]), - reply(From, Fun, {ok, Resp}); - {error, {Code, Msg}, _Metadata} -> - ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) response errcode: ~0p, errmsg: ~0p", - [?CONN_ADAPTER_MOD, Fun, Req, Options, Code, Msg]), - reply(From, Fun, {error, {Code, Msg}}); - {error, Reason} -> - ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) error: ~0p", - [?CONN_ADAPTER_MOD, Fun, Req, Options, Reason]), - reply(From, Fun, {error, Reason}); - {'EXIT', Reason, Stk} -> - ?LOG(error, "CALL ~0p:~0p(~0p, ~0p) throw an exception: ~0p, stacktrace: ~p", - [?CONN_ADAPTER_MOD, Fun, Req, Options, Reason, Stk]), - reply(From, Fun, {error, Reason}) - end, - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. - -terminate(_Reason, _State) -> - ok. - -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -reply(Pid, Fun, Result) -> - Pid ! {hreply, Fun, Result}. diff --git a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl b/apps/emqx_exproto/src/emqx_exproto_gsvr.erl deleted file mode 100644 index 4b5fbd076..000000000 --- a/apps/emqx_exproto/src/emqx_exproto_gsvr.erl +++ /dev/null @@ -1,154 +0,0 @@ -%%-------------------------------------------------------------------- -%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved. -%% -%% Licensed under the Apache License, Version 2.0 (the "License"); -%% you may not use this file except in compliance with the License. -%% You may obtain a copy of the License at -%% -%% http://www.apache.org/licenses/LICENSE-2.0 -%% -%% Unless required by applicable law or agreed to in writing, software -%% distributed under the License is distributed on an "AS IS" BASIS, -%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%% See the License for the specific language governing permissions and -%% limitations under the License. -%%-------------------------------------------------------------------- - -%% The gRPC server for ConnectionAdapter --module(emqx_exproto_gsvr). - --behavior(emqx_exproto_v_1_connection_adapter_bhvr). - --include("emqx_exproto.hrl"). --include_lib("emqx/include/logger.hrl"). - --logger_header("[ExProto gServer]"). - --define(IS_QOS(X), (X =:= 0 orelse X =:= 1 orelse X =:= 2)). - -%% gRPC server callbacks --export([ send/2 - , close/2 - , authenticate/2 - , start_timer/2 - , publish/2 - , subscribe/2 - , unsubscribe/2 - ]). - -%%-------------------------------------------------------------------- -%% gRPC ConnectionAdapter service -%%-------------------------------------------------------------------- - --spec send(ctx:ctx(), emqx_exproto_pb:send_bytes_request()) - -> {ok, emqx_exproto_pb:code_response(), ctx:ctx()} - | grpcbox_stream:grpc_error_response(). -send(Ctx, Req = #{conn := Conn, bytes := Bytes}) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {send, Bytes})), Ctx}. - --spec close(ctx:ctx(), emqx_exproto_pb:close_socket_request()) - -> {ok, emqx_exproto_pb:code_response(), ctx:ctx()} - | grpcbox_stream:grpc_error_response(). -close(Ctx, Req = #{conn := Conn}) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, close)), Ctx}. - --spec authenticate(ctx:ctx(), emqx_exproto_pb:authenticate_request()) - -> {ok, emqx_exproto_pb:code_response(), ctx:ctx()} - | grpcbox_stream:grpc_error_response(). -authenticate(Ctx, Req = #{conn := Conn, - password := Password, - clientinfo := ClientInfo}) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - case validate(clientinfo, ClientInfo) of - false -> - {ok, response({error, ?RESP_REQUIRED_PARAMS_MISSED}), Ctx}; - _ -> - {ok, response(call(Conn, {auth, ClientInfo, Password})), Ctx} - end. - --spec start_timer(ctx:ctx(), emqx_exproto_pb:publish_request()) - -> {ok, emqx_exproto_pb:code_response(), ctx:ctx()} - | grpcbox_stream:grpc_error_response(). -start_timer(Ctx, Req = #{conn := Conn, type := Type, interval := Interval}) - when Type =:= 'KEEPALIVE' andalso Interval > 0 -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {start_timer, keepalive, Interval})), Ctx}; -start_timer(Ctx, Req) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response({error, ?RESP_PARAMS_TYPE_ERROR}), Ctx}. - --spec publish(ctx:ctx(), emqx_exproto_pb:publish_request()) - -> {ok, emqx_exproto_pb:code_response(), ctx:ctx()} - | grpcbox_stream:grpc_error_response(). -publish(Ctx, Req = #{conn := Conn, topic := Topic, qos := Qos, payload := Payload}) - when ?IS_QOS(Qos) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {publish, Topic, Qos, Payload})), Ctx}; - -publish(Ctx, Req) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response({error, ?RESP_PARAMS_TYPE_ERROR}), Ctx}. - --spec subscribe(ctx:ctx(), emqx_exproto_pb:subscribe_request()) - -> {ok, emqx_exproto_pb:code_response(), ctx:ctx()} - | grpcbox_stream:grpc_error_response(). -subscribe(Ctx, Req = #{conn := Conn, topic := Topic, qos := Qos}) - when ?IS_QOS(Qos) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {subscribe, Topic, Qos})), Ctx}; - -subscribe(Ctx, Req) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response({error, ?RESP_PARAMS_TYPE_ERROR}), Ctx}. - --spec unsubscribe(ctx:ctx(), emqx_exproto_pb:unsubscribe_request()) - -> {ok, emqx_exproto_pb:code_response(), ctx:ctx()} - | grpcbox_stream:grpc_error_response(). -unsubscribe(Ctx, Req = #{conn := Conn, topic := Topic}) -> - ?LOG(debug, "Recv ~p function with request ~0p", [?FUNCTION_NAME, Req]), - {ok, response(call(Conn, {unsubscribe, Topic})), Ctx}. - -%%-------------------------------------------------------------------- -%% Internal funcs -%%-------------------------------------------------------------------- - -to_pid(ConnStr) -> - list_to_pid(binary_to_list(ConnStr)). - -call(ConnStr, Req) -> - case catch to_pid(ConnStr) of - {'EXIT', {badarg, _}} -> - {error, ?RESP_PARAMS_TYPE_ERROR, - <<"The conn type error">>}; - Pid when is_pid(Pid) -> - case erlang:is_process_alive(Pid) of - true -> - emqx_exproto_conn:call(Pid, Req); - false -> - {error, ?RESP_CONN_PROCESS_NOT_ALIVE, - <<"Connection process is not alive">>} - end - end. - -%%-------------------------------------------------------------------- -%% Data types - -stringfy(Reason) -> - unicode:characters_to_binary((io_lib:format("~0p", [Reason]))). - -validate(clientinfo, M) -> - Required = [proto_name, proto_ver, clientid], - lists:all(fun(K) -> maps:is_key(K, M) end, Required). - -response(ok) -> - #{code => ?RESP_SUCCESS}; -response({error, Code, Reason}) - when ?IS_GRPC_RESULT_CODE(Code) -> - #{code => Code, message => stringfy(Reason)}; -response({error, Code}) - when ?IS_GRPC_RESULT_CODE(Code) -> - #{code => Code}; -response(Other) -> - #{code => ?RESP_UNKNOWN, message => stringfy(Other)}. diff --git a/apps/emqx_exproto/src/emqx_exproto_sup.erl b/apps/emqx_exproto/src/emqx_exproto_sup.erl index 1ff4b0575..64be2812c 100644 --- a/apps/emqx_exproto/src/emqx_exproto_sup.erl +++ b/apps/emqx_exproto/src/emqx_exproto_sup.erl @@ -20,67 +20,17 @@ -export([start_link/0]). --export([ start_grpc_server/3 - , stop_grpc_server/1 - , start_grpc_client_channel/3 - , stop_grpc_client_channel/1 - ]). - -export([init/1]). -%%-------------------------------------------------------------------- -%% APIs -%%-------------------------------------------------------------------- - start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). --spec start_grpc_server(atom(), inet:port_number(), list()) - -> {ok, pid()} | {error, term()}. -start_grpc_server(Name, Port, SSLOptions) -> - ServerOpts = #{}, - GrpcOpts = #{service_protos => [emqx_exproto_pb], - services => #{'emqx.exproto.v1.ConnectionAdapter' => emqx_exproto_gsvr}}, - ListenOpts = #{port => Port, socket_options => [{reuseaddr, true}]}, - PoolOpts = #{size => 8}, - TransportOpts = maps:from_list(SSLOptions), - Spec = #{id => Name, - start => {grpcbox_services_sup, start_link, - [ServerOpts, GrpcOpts, ListenOpts, - PoolOpts, TransportOpts]}, - type => supervisor, - restart => permanent, - shutdown => infinity}, - supervisor:start_child(?MODULE, Spec). - --spec stop_grpc_server(atom()) -> ok. -stop_grpc_server(Name) -> - ok = supervisor:terminate_child(?MODULE, Name), - ok = supervisor:delete_child(?MODULE, Name). - --spec start_grpc_client_channel( - atom(), - [grpcbox_channel:endpoint()], - grpcbox_channel:options()) -> {ok, pid()} | {error, term()}. -start_grpc_client_channel(Name, Endpoints, Options0) -> - Options = Options0#{sync_start => true}, - Spec = #{id => Name, - start => {grpcbox_channel, start_link, [Name, Endpoints, Options]}, - type => worker}, - supervisor:start_child(?MODULE, Spec). - --spec stop_grpc_client_channel(atom()) -> ok. -stop_grpc_client_channel(Name) -> - ok = supervisor:terminate_child(?MODULE, Name), - ok = supervisor:delete_child(?MODULE, Name). - -%%-------------------------------------------------------------------- -%% Supervisor callbacks -%%-------------------------------------------------------------------- - init([]) -> - %% gRPC Client Pool - PoolSize = emqx_vm:schedulers() * 2, - Pool = emqx_pool_sup:spec([exproto_gcli_pool, hash, PoolSize, - {emqx_exproto_gcli, start_link, []}]), - {ok, {{one_for_one, 10, 5}, [Pool]}}. + DriverMngr = #{id => driver_mngr, + start => {emqx_exproto_driver_mngr, start_link, []}, + restart => permanent, + shutdown => 5000, + type => worker, + modules => [emqx_exproto_driver_mngr]}, + {ok, {{one_for_all, 10, 5}, [DriverMngr]}}. + diff --git a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl index dc6a25c06..5a9d4b830 100644 --- a/apps/emqx_exproto/test/emqx_exproto_SUITE.erl +++ b/apps/emqx_exproto/test/emqx_exproto_SUITE.erl @@ -19,20 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --import(emqx_exproto_echo_svr, - [ frame_connect/2 - , frame_connack/1 - , frame_publish/3 - , frame_puback/1 - , frame_subscribe/2 - , frame_suback/1 - , frame_unsubscribe/1 - , frame_unsuback/1 - , frame_disconnect/0 - ]). - -include_lib("emqx/include/emqx.hrl"). --include_lib("emqx/include/emqx_mqtt.hrl"). -define(TCPOPTS, [binary, {active, false}]). -define(DTLSOPTS, [binary, {active, false}, {protocol, dtls}]). @@ -50,38 +37,48 @@ groups() -> %% @private metrics() -> - [tcp, ssl, udp, dtls]. + [ list_to_atom(X ++ "_" ++ Y) + || X <- ["python3", "java"], Y <- ["tcp", "ssl", "udp", "dtls"]]. -init_per_group(GrpName, Cfg) -> - put(grpname, GrpName), - Svrs = emqx_exproto_echo_svr:start(), +init_per_group(GrpName, Config) -> + [Lang, LisType] = [list_to_atom(X) || X <- string:tokens(atom_to_list(GrpName), "_")], + put(grpname, {Lang, LisType}), emqx_ct_helpers:start_apps([emqx_exproto], fun set_sepecial_cfg/1), - emqx_logger:set_log_level(debug), - [{servers, Svrs}, {listener_type, GrpName} | Cfg]. + [{driver_type, Lang}, + {listener_type, LisType} | Config]. -end_per_group(_, Cfg) -> - emqx_ct_helpers:stop_apps([emqx_exproto]), - emqx_exproto_echo_svr:stop(proplists:get_value(servers, Cfg)). +end_per_group(_, _) -> + emqx_ct_helpers:stop_apps([emqx_exproto]). set_sepecial_cfg(emqx_exproto) -> - LisType = get(grpname), + {Lang, LisType} = get(grpname), + Path = emqx_ct_helpers:deps_path(emqx_exproto, "example/"), Listeners = application:get_env(emqx_exproto, listeners, []), + Driver = compile(Lang, Path), SockOpts = socketopts(LisType), UpgradeOpts = fun(Opts) -> - Opts2 = lists:keydelete(tcp_options, 1, Opts), + Opts1 = lists:keydelete(driver, 1, Opts), + Opts2 = lists:keydelete(tcp_options, 1, Opts1), Opts3 = lists:keydelete(ssl_options, 1, Opts2), Opts4 = lists:keydelete(udp_options, 1, Opts3), Opts5 = lists:keydelete(dtls_options, 1, Opts4), - SockOpts ++ Opts5 + Driver ++ SockOpts ++ Opts5 end, NListeners = [{Proto, LisType, LisOn, UpgradeOpts(Opts)} || {Proto, _Type, LisOn, Opts} <- Listeners], application:set_env(emqx_exproto, listeners, NListeners); -set_sepecial_cfg(emqx) -> - application:set_env(emqx, allow_anonymous, true), - application:set_env(emqx, enable_acl_cache, false), +set_sepecial_cfg(_App) -> ok. +compile(java, Path) -> + ErlPortJar = emqx_ct_helpers:deps_path(erlport, "priv/java/_pkgs/erlport.jar"), + ct:pal(os:cmd(lists:concat(["cd ", Path, " && ", + "rm -rf Main.class State.class && ", + "javac -cp ", ErlPortJar, " Main.java"]))), + [{driver, [{type, java}, {path, Path}, {cbm, 'Main'}]}]; +compile(python3, Path) -> + [{driver, [{type, python3}, {path, Path}, {cbm, main}]}]. + %%-------------------------------------------------------------------- %% Tests cases %%-------------------------------------------------------------------- @@ -89,263 +86,24 @@ set_sepecial_cfg(emqx) -> t_start_stop(_) -> ok. -t_mountpoint_echo(Cfg) -> +t_echo(Cfg) -> SockType = proplists:get_value(listener_type, Cfg), + Bin = rand_bytes(), + Sock = open(SockType), - Client = #{proto_name => <<"demo">>, - proto_ver => <<"v0.1">>, - clientid => <<"test_client_1">>, - mountpoint => <<"ct/">> - }, - Password = <<"123456">>, + send(Sock, Bin), - ConnBin = frame_connect(Client, Password), - ConnAckBin = frame_connack(0), - - send(Sock, ConnBin), - {ok, ConnAckBin} = recv(Sock, 5000), - - SubBin = frame_subscribe(<<"t/#">>, 1), - SubAckBin = frame_suback(0), - - send(Sock, SubBin), - {ok, SubAckBin} = recv(Sock, 5000), - - emqx:publish(emqx_message:make(<<"ct/t/dn">>, <<"echo">>)), - PubBin1 = frame_publish(<<"t/dn">>, 0, <<"echo">>), - {ok, PubBin1} = recv(Sock, 5000), - - PubBin2 = frame_publish(<<"t/up">>, 0, <<"echo">>), - PubAckBin = frame_puback(0), - - emqx:subscribe(<<"ct/t/up">>), - - send(Sock, PubBin2), - {ok, PubAckBin} = recv(Sock, 5000), - - receive - {deliver, _, _} -> ok - after 1000 -> - error(echo_not_running) - end, - close(Sock). - -t_auth_deny(Cfg) -> - SockType = proplists:get_value(listener_type, Cfg), - Sock = open(SockType), - - Client = #{proto_name => <<"demo">>, - proto_ver => <<"v0.1">>, - clientid => <<"test_client_1">> - }, - Password = <<"123456">>, - - ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), - ok = meck:expect(emqx_access_control, authenticate, - fun(_) -> {error, ?RC_NOT_AUTHORIZED} end), - - ConnBin = frame_connect(Client, Password), - ConnAckBin = frame_connack(1), - - send(Sock, ConnBin), - {ok, ConnAckBin} = recv(Sock, 5000), - - SockType =/= udp andalso begin - {error, closed} = recv(Sock, 5000) - end, - meck:unload([emqx_access_control]). - -t_acl_deny(Cfg) -> - SockType = proplists:get_value(listener_type, Cfg), - Sock = open(SockType), - - Client = #{proto_name => <<"demo">>, - proto_ver => <<"v0.1">>, - clientid => <<"test_client_1">> - }, - Password = <<"123456">>, - - ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]), - ok = meck:expect(emqx_access_control, check_acl, fun(_, _, _) -> deny end), - - ConnBin = frame_connect(Client, Password), - ConnAckBin = frame_connack(0), - - send(Sock, ConnBin), - {ok, ConnAckBin} = recv(Sock, 5000), - - SubBin = frame_subscribe(<<"t/#">>, 1), - SubAckBin = frame_suback(1), - - send(Sock, SubBin), - {ok, SubAckBin} = recv(Sock, 5000), + {ok, Bin} = recv(Sock, byte_size(Bin), 5000), + %% pubsub echo + emqx:subscribe(<<"t/#">>), emqx:publish(emqx_message:make(<<"t/dn">>, <<"echo">>)), + First = receive {_, _, X} -> X#message.payload end, + First = receive {_, _, Y} -> Y#message.payload end, - PubBin = frame_publish(<<"t/dn">>, 0, <<"echo">>), - PubBinFailedAck = frame_puback(1), - PubBinSuccesAck = frame_puback(0), - - send(Sock, PubBin), - {ok, PubBinFailedAck} = recv(Sock, 5000), - - meck:unload([emqx_access_control]), - - send(Sock, PubBin), - {ok, PubBinSuccesAck} = recv(Sock, 5000), close(Sock). -t_keepalive_timeout(Cfg) -> - SockType = proplists:get_value(listener_type, Cfg), - Sock = open(SockType), - - Client = #{proto_name => <<"demo">>, - proto_ver => <<"v0.1">>, - clientid => <<"test_client_1">>, - keepalive => 2 - }, - Password = <<"123456">>, - - ConnBin = frame_connect(Client, Password), - ConnAckBin = frame_connack(0), - - send(Sock, ConnBin), - {ok, ConnAckBin} = recv(Sock, 5000), - - DisconnectBin = frame_disconnect(), - {ok, DisconnectBin} = recv(Sock, 10000), - - SockType =/= udp andalso begin - {error, closed} = recv(Sock, 5000) - end, ok. - -t_hook_connected_disconnected(Cfg) -> - SockType = proplists:get_value(listener_type, Cfg), - Sock = open(SockType), - - Client = #{proto_name => <<"demo">>, - proto_ver => <<"v0.1">>, - clientid => <<"test_client_1">> - }, - Password = <<"123456">>, - - ConnBin = frame_connect(Client, Password), - ConnAckBin = frame_connack(0), - - Parent = self(), - HookFun1 = fun(_, _) -> Parent ! connected, ok end, - HookFun2 = fun(_, _, _) -> Parent ! disconnected, ok end, - emqx:hook('client.connected', HookFun1), - emqx:hook('client.disconnected', HookFun2), - - - send(Sock, ConnBin), - {ok, ConnAckBin} = recv(Sock, 5000), - - receive - connected -> ok - after 1000 -> - error(hook_is_not_running) - end, - - DisconnectBin = frame_disconnect(), - send(Sock, DisconnectBin), - - receive - disconnected -> ok - after 1000 -> - error(hook_is_not_running) - end, - - SockType =/= udp andalso begin - {error, closed} = recv(Sock, 5000) - end, - emqx:unhook('client.connected', HookFun1), - emqx:unhook('client.disconnected', HookFun2). - -t_hook_session_subscribed_unsubscribed(Cfg) -> - SockType = proplists:get_value(listener_type, Cfg), - Sock = open(SockType), - - Client = #{proto_name => <<"demo">>, - proto_ver => <<"v0.1">>, - clientid => <<"test_client_1">> - }, - Password = <<"123456">>, - - ConnBin = frame_connect(Client, Password), - ConnAckBin = frame_connack(0), - - send(Sock, ConnBin), - {ok, ConnAckBin} = recv(Sock, 5000), - - Parent = self(), - HookFun1 = fun(_, _, _) -> Parent ! subscribed, ok end, - HookFun2 = fun(_, _, _) -> Parent ! unsubscribed, ok end, - emqx:hook('session.subscribed', HookFun1), - emqx:hook('session.unsubscribed', HookFun2), - - SubBin = frame_subscribe(<<"t/#">>, 1), - SubAckBin = frame_suback(0), - - send(Sock, SubBin), - {ok, SubAckBin} = recv(Sock, 5000), - - receive - subscribed -> ok - after 1000 -> - error(hook_is_not_running) - end, - - UnsubBin = frame_unsubscribe(<<"t/#">>), - UnsubAckBin = frame_unsuback(0), - - send(Sock, UnsubBin), - {ok, UnsubAckBin} = recv(Sock, 5000), - - receive - unsubscribed -> ok - after 1000 -> - error(hook_is_not_running) - end, - - close(Sock), - emqx:unhook('session.subscribed', HookFun1), - emqx:unhook('session.unsubscribed', HookFun2). - -t_hook_message_delivered(Cfg) -> - SockType = proplists:get_value(listener_type, Cfg), - Sock = open(SockType), - - Client = #{proto_name => <<"demo">>, - proto_ver => <<"v0.1">>, - clientid => <<"test_client_1">> - }, - Password = <<"123456">>, - - ConnBin = frame_connect(Client, Password), - ConnAckBin = frame_connack(0), - - send(Sock, ConnBin), - {ok, ConnAckBin} = recv(Sock, 5000), - - SubBin = frame_subscribe(<<"t/#">>, 1), - SubAckBin = frame_suback(0), - - send(Sock, SubBin), - {ok, SubAckBin} = recv(Sock, 5000), - - HookFun1 = fun(_, Msg) -> {ok, Msg#message{payload = <<"2">>}} end, - emqx:hook('message.delivered', HookFun1), - - emqx:publish(emqx_message:make(<<"t/dn">>, <<"1">>)), - PubBin1 = frame_publish(<<"t/dn">>, 0, <<"2">>), - {ok, PubBin1} = recv(Sock, 5000), - - close(Sock), - emqx:unhook('message.delivered', HookFun1). - %%-------------------------------------------------------------------- %% Utils @@ -379,15 +137,15 @@ send({ssl, Sock}, Bin) -> send({dtls, Sock}, Bin) -> ssl:send(Sock, Bin). -recv({tcp, Sock}, Ts) -> - gen_tcp:recv(Sock, 0, Ts); -recv({udp, Sock}, Ts) -> - {ok, {_, _, Bin}} = gen_udp:recv(Sock, 0, Ts), +recv({tcp, Sock}, Size, Ts) -> + gen_tcp:recv(Sock, Size, Ts); +recv({udp, Sock}, Size, Ts) -> + {ok, {_, _, Bin}} = gen_udp:recv(Sock, Size, Ts), {ok, Bin}; -recv({ssl, Sock}, Ts) -> - ssl:recv(Sock, 0, Ts); -recv({dtls, Sock}, Ts) -> - ssl:recv(Sock, 0, Ts). +recv({ssl, Sock}, Size, Ts) -> + ssl:recv(Sock, Size, Ts); +recv({dtls, Sock}, Size, Ts) -> + ssl:recv(Sock, Size, Ts). close({tcp, Sock}) -> gen_tcp:close(Sock); diff --git a/apps/emqx_lua_hook/rebar.config b/apps/emqx_lua_hook/rebar.config index 7a5e7ea5c..fe4fa19cb 100644 --- a/apps/emqx_lua_hook/rebar.config +++ b/apps/emqx_lua_hook/rebar.config @@ -1,3 +1,3 @@ {deps, [{luerl, {git, "https://github.com/emqx/luerl", {tag, "v0.3.1"}}} - ]}. \ No newline at end of file + ]}. diff --git a/apps/emqx_lwm2m/rebar.config b/apps/emqx_lwm2m/rebar.config index 6b72537b7..d82966e71 100644 --- a/apps/emqx_lwm2m/rebar.config +++ b/apps/emqx_lwm2m/rebar.config @@ -1,3 +1,3 @@ {deps, - [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v1.1.0"}}} - ]}. \ No newline at end of file + [{lwm2m_coap, {git, "https://github.com/emqx/lwm2m-coap", {tag, "v1.1.1"}}} + ]}. diff --git a/apps/emqx_management/src/emqx_mgmt.erl b/apps/emqx_management/src/emqx_mgmt.erl index 36d4b8e73..b5a319278 100644 --- a/apps/emqx_management/src/emqx_mgmt.erl +++ b/apps/emqx_management/src/emqx_mgmt.erl @@ -457,7 +457,9 @@ do_subscribe(ClientId, TopicTables) -> end. %%TODO: ??? -publish(Msg) -> emqx:publish(Msg). +publish(Msg) -> + emqx_metrics:inc_msg(Msg), + emqx:publish(Msg). unsubscribe(ClientId, Topic) -> unsubscribe(ekka_mnesia:running_nodes(), ClientId, Topic). diff --git a/apps/emqx_management/src/emqx_mgmt_api_data.erl b/apps/emqx_management/src/emqx_mgmt_api_data.erl index 6fc44add0..a449141ce 100644 --- a/apps/emqx_management/src/emqx_mgmt_api_data.erl +++ b/apps/emqx_management/src/emqx_mgmt_api_data.erl @@ -70,6 +70,10 @@ , delete/2 ]). +-export([ get_list_exported/0 + , do_import/1 + ]). + export(_Bindings, _Params) -> Rules = emqx_mgmt:export_rules(), Resources = emqx_mgmt:export_resources(), @@ -97,8 +101,11 @@ export(_Bindings, _Params) -> {auth_username, AuthUsername}, {auth_mnesia, AuthMnesia}, {acl_mnesia, AclMnesia}, - {schemas, Schemas}], + {schemas, Schemas} + ], + Bin = emqx_json:encode(Data), + ok = filelib:ensure_dir(NFilename), case file:write_file(NFilename, Bin) of ok -> case file:read_file_info(NFilename) of @@ -106,7 +113,9 @@ export(_Bindings, _Params) -> CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), return({ok, [{filename, list_to_binary(Filename)}, {size, Size}, - {created_at, list_to_binary(CreatedAt)}]}); + {created_at, list_to_binary(CreatedAt)}, + {node, node()} + ]}); {error, Reason} -> return({error, Reason}) end; @@ -115,66 +124,86 @@ export(_Bindings, _Params) -> end. list_exported(_Bindings, _Params) -> + List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ], + NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, lists:append(List))), + return({ok, NList}). + +get_list_exported() -> Dir = emqx:get_env(data_dir), {ok, Files} = file:list_dir_all(Dir), - List = lists:foldl(fun(File, Acc) -> - case filename:extension(File) =:= ".json" of - true -> - FullFile = filename:join([Dir, File]), - case file:read_file_info(FullFile) of - {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> - CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), - Seconds = calendar:datetime_to_gregorian_seconds(CTime), - [{Seconds, [{filename, list_to_binary(File)}, - {size, Size}, - {created_at, list_to_binary(CreatedAt)}]} | Acc]; - {error, Reason} -> - logger:error("Read file info of ~s failed with: ~p", [File, Reason]), - Acc - end; - false -> - Acc - end - end, [], Files), - NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, List)), - return({ok, NList}). + lists:foldl( + fun(File, Acc) -> + case filename:extension(File) =:= ".json" of + true -> + FullFile = filename:join([Dir, File]), + case file:read_file_info(FullFile) of + {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} -> + CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]), + Seconds = calendar:datetime_to_gregorian_seconds(CTime), + [{Seconds, [{filename, list_to_binary(File)}, + {size, Size}, + {created_at, list_to_binary(CreatedAt)}, + {node, node()} + ]} | Acc]; + {error, Reason} -> + logger:error("Read file info of ~s failed with: ~p", [File, Reason]), + Acc + end; + false -> Acc + end + end, [], Files). import(_Bindings, Params) -> case proplists:get_value(<<"filename">>, Params) of undefined -> return({error, missing_required_params}); Filename -> - FullFilename = filename:join([emqx:get_env(data_dir), Filename]), - case file:read_file(FullFilename) of - {ok, Json} -> - Data = emqx_json:decode(Json, [return_maps]), - Version = emqx_mgmt:to_version(maps:get(<<"version">>, Data)), - case lists:member(Version, ?VERSIONS) of - true -> - try - emqx_mgmt:import_resources(maps:get(<<"resources">>, Data, [])), - emqx_mgmt:import_rules(maps:get(<<"rules">>, Data, [])), - emqx_mgmt:import_blacklist(maps:get(<<"blacklist">>, Data, [])), - emqx_mgmt:import_applications(maps:get(<<"apps">>, Data, [])), - emqx_mgmt:import_users(maps:get(<<"users">>, Data, [])), - emqx_mgmt:import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), - emqx_mgmt:import_auth_username(maps:get(<<"auth_username">>, Data, [])), - emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])), - emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])), - emqx_mgmt:import_schemas(maps:get(<<"schemas">>, Data, [])), - logger:debug("The emqx data has been imported successfully"), - return() - catch Class:Reason:Stack -> - logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), - return({error, import_failed}) - end; - false -> - logger:error("Unsupported version: ~p", [Version]), - return({error, unsupported_version}) + Result = case proplists:get_value(<<"node">>, Params) of + undefined -> do_import(Filename); + Node -> + case lists:member(Node, + [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ] + ) of + true -> rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename]); + false -> return({error, no_existent_node}) + end + end, + return(Result) + end. + +do_import(Filename) -> + FullFilename = filename:join([emqx:get_env(data_dir), Filename]), + case file:read_file(FullFilename) of + {ok, Json} -> + Data = emqx_json:decode(Json, [return_maps]), + Version = emqx_mgmt:to_version(maps:get(<<"version">>, Data)), + case lists:member(Version, ?VERSIONS) of + true -> + try + emqx_mgmt:import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])), + emqx_mgmt:import_resources(maps:get(<<"resources">>, Data, [])), + emqx_mgmt:import_rules(maps:get(<<"rules">>, Data, [])), + emqx_mgmt:import_blacklist(maps:get(<<"blacklist">>, Data, [])), + emqx_mgmt:import_applications(maps:get(<<"apps">>, Data, [])), + emqx_mgmt:import_users(maps:get(<<"users">>, Data, [])), + emqx_mgmt:import_modules(maps:get(<<"modules">>, Data, [])), + emqx_mgmt:import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])), + emqx_mgmt:import_auth_username(maps:get(<<"auth_username">>, Data, [])), + emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version), + emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version), + emqx_mgmt:import_schemas(maps:get(<<"schemas">>, Data, [])), + logger:debug("The emqx data has been imported successfully"), + ok + catch Class:Reason:Stack -> + logger:error("The emqx data import failed: ~0p", [{Class,Reason,Stack}]), + {error, import_failed} end; - {error, Reason} -> - return({error, Reason}) - end + false -> + logger:error("Unsupported version: ~p", [Version]), + {error, unsupported_version} + end; + {error, Reason} -> + {error, Reason} end. download(#{filename := Filename}, _Params) -> @@ -195,7 +224,7 @@ do_upload(_Bindings, #{<<"filename">> := Filename, FullFilename = filename:join([emqx:get_env(data_dir), Filename]), case file:write_file(FullFilename, Bin) of ok -> - return(); + return({ok, [{node, node()}]}); {error, Reason} -> return({error, Reason}) end; @@ -214,4 +243,4 @@ delete(#{filename := Filename}, _Params) -> return(); {error, Reason} -> return({error, Reason}) - end. \ No newline at end of file + end. diff --git a/apps/emqx_management/src/emqx_mgmt_cli.erl b/apps/emqx_management/src/emqx_mgmt_cli.erl index ae9b16459..b0870cbf3 100644 --- a/apps/emqx_management/src/emqx_mgmt_cli.erl +++ b/apps/emqx_management/src/emqx_mgmt_cli.erl @@ -585,6 +585,7 @@ data(["export"]) -> {auth_mnesia, AuthMnesia}, {acl_mnesia, AclMnesia}, {schemas, Schemas}], + ok = filelib:ensure_dir(NFilename), case file:write_file(NFilename, emqx_json:encode(Data)) of ok -> emqx_ctl:print("The emqx data has been successfully exported to ~s.~n", [NFilename]); diff --git a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl index 2b39b95cb..02ab1e8fd 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_SUITE.erl @@ -53,7 +53,8 @@ groups() -> acl_cache, pubsub, routes_and_subscriptions, - stats]}]. + stats, + data]}]. init_per_suite(Config) -> emqx_ct_helpers:start_apps([emqx, emqx_management, emqx_reloader]), @@ -65,6 +66,36 @@ end_per_suite(_Config) -> emqx_ct_helpers:stop_apps([emqx_reloader, emqx_management, emqx]), ekka_mnesia:ensure_stopped(). +init_per_testcase(data, Config) -> + ok = emqx_dashboard_admin:mnesia(boot), + application:ensure_all_started(emqx_dahboard), + ok = emqx_rule_registry:mnesia(boot), + application:ensure_all_started(emqx_rule_engine), + + meck:new(emqx_sys, [passthrough, no_history]), + meck:expect(emqx_sys, version, 0, + fun() -> + Tag =os:cmd("git describe --abbrev=0 --tags") -- "\n", + re:replace(Tag, "[v|e]", "", [{return ,list}]) + end), + + Config; + +init_per_testcase(_, Config) -> + Config. + +stop_pre_testcase(data, _Config) -> + application:stop(emqx_dahboard), + application:stop(emqx_rule_engine), + application:stop(emqx_modules), + application:stop(emqx_schema_registry), + application:stop(emqx_conf), + meck:unload(emqx_sys), + ok; + +stop_pre_testcase(_, _Config) -> + ok. + get(Key, ResponseBody) -> maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])). @@ -432,6 +463,10 @@ acl_cache(_) -> ok = emqtt:disconnect(C1). pubsub(_) -> + Qos1Received = emqx_metrics:val('messages.qos1.received'), + Qos2Received = emqx_metrics:val('messages.qos2.received'), + Received = emqx_metrics:val('messages.received'), + ClientId = <<"client1">>, Options = #{clientid => ClientId, proto_ver => 5}, @@ -532,7 +567,11 @@ pubsub(_) -> {ok, Data3} = request_api(post, api_path(["mqtt/unsubscribe_batch"]), [], auth_header_(), Body3), loop(maps:get(<<"data">>, jiffy:decode(list_to_binary(Data3), [return_maps]))), - ok = emqtt:disconnect(C1). + ok = emqtt:disconnect(C1), + + ?assertEqual(2, emqx_metrics:val('messages.qos1.received') - Qos1Received), + ?assertEqual(2, emqx_metrics:val('messages.qos2.received') - Qos2Received), + ?assertEqual(4, emqx_metrics:val('messages.received') - Received). loop([]) -> []; @@ -596,6 +635,17 @@ stats(_) -> ?assertEqual(<<"undefined">>, get(<<"message">>, Return)), meck:unload(emqx_mgmt). +data(_) -> + {ok, Data} = request_api(post, api_path(["data","export"]), [], auth_header_(), [#{}]), + #{<<"filename">> := Filename, <<"node">> := Node} = emqx_ct_http:get_http_data(Data), + {ok, DataList} = request_api(get, api_path(["data","export"]), auth_header_()), + ?assertEqual(true, lists:member(emqx_ct_http:get_http_data(Data), emqx_ct_http:get_http_data(DataList))), + + ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename, <<"node">> => Node})), + ?assertMatch({ok, _}, request_api(post, api_path(["data","import"]), [], auth_header_(), #{<<"filename">> => Filename})), + + ok. + request_api(Method, Url, Auth) -> request_api(Method, Url, [], Auth, []). diff --git a/apps/emqx_passwd/rebar.config b/apps/emqx_passwd/rebar.config index acaa113b3..ad28a5bff 100644 --- a/apps/emqx_passwd/rebar.config +++ b/apps/emqx_passwd/rebar.config @@ -1,7 +1,7 @@ -{deps, [ - {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {tag, "0.6.0"}}} -]}. +{deps, + [{pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.3"}}} + ]}. {plugins, [ {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}} - ]}. \ No newline at end of file + ]}. diff --git a/apps/emqx_passwd/src/emqx_passwd.app.src b/apps/emqx_passwd/src/emqx_passwd.app.src index 7d384a2ad..8e66d443a 100644 --- a/apps/emqx_passwd/src/emqx_passwd.app.src +++ b/apps/emqx_passwd/src/emqx_passwd.app.src @@ -1,9 +1,9 @@ {application, emqx_passwd, [{description, "Password Hash Library for EMQ X Broker"}, - {vsn, "4.3.0"}, % strict semver, bump manually! + {vsn, "0.1.1"}, % strict semver, bump manually! {modules, ['emqx_passwd']}, {registered, []}, - {applications, [kernel,stdlib,ssl,pbkdf2,bcrypt,emqx]}, + {applications, [kernel,stdlib,ssl,pbkdf2,emqx]}, {env, []}, {licenses, ["Apache-2.0"]}, {maintainers, ["EMQ X Team "]}, diff --git a/apps/emqx_prometheus/rebar.config b/apps/emqx_prometheus/rebar.config index 0f8b760c1..2d8d63134 100644 --- a/apps/emqx_prometheus/rebar.config +++ b/apps/emqx_prometheus/rebar.config @@ -1,3 +1,3 @@ {deps, [{prometheus, {git, "https://github.com/emqx/prometheus.erl", {tag, "v3.1.1"}}} - ]}. \ No newline at end of file + ]}. diff --git a/apps/emqx_psk_file/rebar.config b/apps/emqx_psk_file/rebar.config index 7b1f06cca..7b30a8fd8 100644 --- a/apps/emqx_psk_file/rebar.config +++ b/apps/emqx_psk_file/rebar.config @@ -1 +1 @@ -{deps, []}. \ No newline at end of file +{deps, []}. diff --git a/apps/emqx_recon/rebar.config b/apps/emqx_recon/rebar.config index a60702ae3..f5fd83abe 100644 --- a/apps/emqx_recon/rebar.config +++ b/apps/emqx_recon/rebar.config @@ -1,3 +1,3 @@ {deps, [ {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.0"}}} -]}. \ No newline at end of file +]}. diff --git a/apps/emqx_retainer/rebar.config b/apps/emqx_retainer/rebar.config index 7b1f06cca..7b30a8fd8 100644 --- a/apps/emqx_retainer/rebar.config +++ b/apps/emqx_retainer/rebar.config @@ -1 +1 @@ -{deps, []}. \ No newline at end of file +{deps, []}. diff --git a/apps/emqx_rule_engine/rebar.config b/apps/emqx_rule_engine/rebar.config index 7b1f06cca..7b30a8fd8 100644 --- a/apps/emqx_rule_engine/rebar.config +++ b/apps/emqx_rule_engine/rebar.config @@ -1 +1 @@ -{deps, []}. \ No newline at end of file +{deps, []}. diff --git a/apps/emqx_rule_engine/src/emqx_rule_actions.erl b/apps/emqx_rule_engine/src/emqx_rule_actions.erl index 5ee860e94..e024f0cfa 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_actions.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_actions.erl @@ -134,35 +134,37 @@ on_action_create_republish(Id, #{<<"target_topic">> := TargetTopic, <<"target_qo (Selected, _Envs = #{qos := QoS, flags := Flags, timestamp := Timestamp}) -> ?LOG(debug, "[republish] republish to: ~p, Payload: ~p", [TargetTopic, Selected]), - emqx_broker:safe_publish( - emqx_message:set_headers( - #{republish_by => Id}, - #message{ - id = emqx_guid:gen(), - qos = if TargetQoS =:= -1 -> QoS; true -> TargetQoS end, - from = Id, - flags = Flags, - topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected), - payload = emqx_rule_utils:proc_tmpl(PayloadTks, Selected), - timestamp = Timestamp - }) - ); + increase_and_publish( + #message{ + id = emqx_guid:gen(), + qos = if TargetQoS =:= -1 -> QoS; true -> TargetQoS end, + from = Id, + flags = Flags, + headers = #{republish_by => Id}, + topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected), + payload = emqx_rule_utils:proc_tmpl(PayloadTks, Selected), + timestamp = Timestamp + }); %% in case this is not a "message.publish" request (Selected, _Envs) -> ?LOG(debug, "[republish] republish to: ~p, Payload: ~p", [TargetTopic, Selected]), - emqx_broker:safe_publish( - #message{ - id = emqx_guid:gen(), - qos = if TargetQoS =:= -1 -> 0; true -> TargetQoS end, - from = Id, - flags = #{dup => false, retain => false}, - headers = #{republish_by => Id}, - topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected), - payload = emqx_rule_utils:proc_tmpl(PayloadTks, Selected), - timestamp = erlang:system_time(millisecond) - }) + increase_and_publish( + #message{ + id = emqx_guid:gen(), + qos = if TargetQoS =:= -1 -> 0; true -> TargetQoS end, + from = Id, + flags = #{dup => false, retain => false}, + headers = #{republish_by => Id}, + topic = emqx_rule_utils:proc_tmpl(TopicTks, Selected), + payload = emqx_rule_utils:proc_tmpl(PayloadTks, Selected), + timestamp = erlang:system_time(millisecond) + }) end. +increase_and_publish(Msg) -> + emqx_metrics:inc_msg(Msg), + emqx_broker:safe_publish(Msg). + on_action_do_nothing(_, _) -> fun(_, _) -> ok end. diff --git a/apps/emqx_rule_engine/src/emqx_rule_engine.erl b/apps/emqx_rule_engine/src/emqx_rule_engine.erl index 41b961230..8384ef4db 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_engine.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_engine.erl @@ -420,7 +420,7 @@ action_instance_id(ActionName) -> iolist_to_binary([atom_to_list(ActionName), "_", integer_to_list(erlang:system_time())]). cluster_call(Func, Args) -> - case rpc:multicall([node() | nodes()], ?MODULE, Func, Args, 5000) of + case rpc:multicall(ekka_mnesia:running_nodes(), ?MODULE, Func, Args, 5000) of {ResL, []} -> case lists:filter(fun(ok) -> false; (_) -> true end, ResL) of [] -> ok; diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index 66eafc36e..6795f1522 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -130,10 +130,18 @@ ]). %% Map Funcs +-export([ map_new/0 + ]). + -export([ map_get/2 , map_get/3 , map_put/3 - , map_new/0 + ]). + +%% For backword compatibility +-export([ mget/2 + , mget/3 + , mput/3 ]). %% Array Funcs @@ -225,7 +233,7 @@ payload() -> payload(Path) -> fun(#{payload := Payload}) when erlang:is_map(Payload) -> - emqx_rule_maps:nested_get(map_path(Path), Payload); + map_get(Path, Payload); (_) -> undefined end. @@ -599,6 +607,15 @@ map_get(Key, Map) -> map_get(Key, Map, undefined). map_get(Key, Map, Default) -> + emqx_rule_maps:nested_get(map_path(Key), Map, Default). + +map_put(Key, Val, Map) -> + emqx_rule_maps:nested_put(map_path(Key), Val, Map). + +mget(Key, Map) -> + mget(Key, Map, undefined). + +mget(Key, Map, Default) -> case maps:find(Key, Map) of {ok, Val} -> Val; error when is_atom(Key) -> @@ -622,7 +639,7 @@ map_get(Key, Map, Default) -> Default end. -map_put(Key, Val, Map) -> +mput(Key, Val, Map) -> case maps:find(Key, Map) of {ok, _} -> maps:put(Key, Val, Map); error when is_atom(Key) -> diff --git a/apps/emqx_rule_engine/src/emqx_rule_validator.erl b/apps/emqx_rule_engine/src/emqx_rule_validator.erl index 8acbb3afc..76c5ea19c 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_validator.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_validator.erl @@ -25,7 +25,7 @@ -type(params_spec() :: #{atom() => term()}). -type(params() :: #{binary() => term()}). --define(DATA_TYPES, [string, number, float, boolean, object, array]). +-define(DATA_TYPES, [string, number, float, boolean, object, array, file]). %%------------------------------------------------------------------------------ %% APIs @@ -68,6 +68,8 @@ do_validate_param(Val, Spec = #{type := Type}) -> end, validate_type(Val, Type, Spec). +validate_type(Val, file, _Spec) -> + ok = validate_file(Val); validate_type(Val, string, Spec) -> ok = validate_string(Val, reg_exp(maps:get(format, Spec, any))); validate_type(Val, number, Spec) -> @@ -110,6 +112,9 @@ validate_boolean(true) -> ok; validate_boolean(false) -> ok; validate_boolean(Val) -> error({invalid_data_type, {boolean, Val}}). +validate_file(Val) when is_binary(Val) -> ok; +validate_file(Val) -> error({invalid_data_type, {file, Val}}). + reg_exp(url) -> "^https?://\\w+(\.\\w+)*(:[0-9]+)?"; reg_exp(topic) -> "^/?(\\w|\\#|\\+)+(/?(\\w|\\#|\\+))*/?$"; reg_exp(resource_type) -> "[a-zA-Z0-9_:-]"; diff --git a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl index 9836b96f0..348ef046f 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl @@ -51,7 +51,7 @@ groups() -> ]}, {actions, [], [t_inspect_action - %,t_republish_action + ,t_republish_action ]}, {api, [], [t_crud_rule_api, @@ -325,12 +325,14 @@ t_inspect_action(_Config) -> ok. t_republish_action(_Config) -> + Qos0Received = emqx_metrics:val('messages.qos0.received'), + Received = emqx_metrics:val('messages.received'), ok = emqx_rule_engine:load_providers(), {ok, #rule{id = Id, for = [<<"t1">>]}} = emqx_rule_engine:create_rule( #{rawsql => <<"select topic, payload, qos from \"t1\"">>, actions => [#{name => 'republish', - args => #{<<"target_topic">> => <<"t1">>, + args => #{<<"target_topic">> => <<"t2">>, <<"target_qos">> => -1, <<"payload_tmpl">> => <<"${payload}">>}}], description => <<"builtin-republish-rule">>}), @@ -347,6 +349,8 @@ t_republish_action(_Config) -> end, emqtt:stop(Client), emqx_rule_registry:remove_rule(Id), + ?assertEqual(2, emqx_metrics:val('messages.qos0.received') - Qos0Received ), + ?assertEqual(2, emqx_metrics:val('messages.received') - Received), ok. %%------------------------------------------------------------------------------ diff --git a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl index 8a49ab520..582eb5832 100644 --- a/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl +++ b/apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl @@ -489,12 +489,27 @@ t_contains(_) -> t_map_get(_) -> ?assertEqual(1, apply_func(map_get, [<<"a">>, #{a => 1}])), - ?assertEqual(undefined, apply_func(map_get, [<<"a">>, #{}])). + ?assertEqual(undefined, apply_func(map_get, [<<"a">>, #{}])), + ?assertEqual(1, apply_func(map_get, [<<"a.b">>, #{a => #{b => 1}}])), + ?assertEqual(undefined, apply_func(map_get, [<<"a.c">>, #{a => #{b => 1}}])). t_map_put(_) -> ?assertEqual(#{<<"a">> => 1}, apply_func(map_put, [<<"a">>, 1, #{}])), + ?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])), + ?assertEqual(#{<<"a">> => #{<<"b">> => 1}}, apply_func(map_put, [<<"a.b">>, 1, #{}])), + ?assertEqual(#{a => #{b => 1, <<"c">> => 1}}, apply_func(map_put, [<<"a.c">>, 1, #{a => #{b => 1}}])). + + t_mget(_) -> + ?assertEqual(1, apply_func(map_get, [<<"a">>, #{a => 1}])), + ?assertEqual(1, apply_func(map_get, [<<"a">>, #{<<"a">> => 1}])), + ?assertEqual(undefined, apply_func(map_get, [<<"a">>, #{}])). + + t_mput(_) -> + ?assertEqual(#{<<"a">> => 1}, apply_func(map_put, [<<"a">>, 1, #{}])), + ?assertEqual(#{<<"a">> => 2}, apply_func(map_put, [<<"a">>, 2, #{<<"a">> => 1}])), ?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])). + %%------------------------------------------------------------------------------ %% Test cases for Hash funcs %%------------------------------------------------------------------------------ diff --git a/apps/emqx_sasl/rebar.config b/apps/emqx_sasl/rebar.config index 7b1f06cca..7b30a8fd8 100644 --- a/apps/emqx_sasl/rebar.config +++ b/apps/emqx_sasl/rebar.config @@ -1 +1 @@ -{deps, []}. \ No newline at end of file +{deps, []}. diff --git a/apps/emqx_sasl/src/emqx_sasl_api.erl b/apps/emqx_sasl/src/emqx_sasl_api.erl index e17296285..1f77cd9e7 100644 --- a/apps/emqx_sasl/src/emqx_sasl_api.erl +++ b/apps/emqx_sasl/src/emqx_sasl_api.erl @@ -92,8 +92,8 @@ get(_Bindings, #{<<"mechanism">> := Mechanism0, case Mechanism of <<"SCRAM-SHA-1">> -> case emqx_sasl_scram:lookup(Username) of - {ok, AuthInfo} -> - return({ok, AuthInfo}); + {ok, AuthInfo = #{salt := Salt}} -> + return({ok, AuthInfo#{salt => base64:decode(Salt)}}); {error, Reason} -> return({error, Reason}) end; diff --git a/apps/emqx_sasl/src/emqx_sasl_cli.erl b/apps/emqx_sasl/src/emqx_sasl_cli.erl index 19736a108..375fc60ec 100644 --- a/apps/emqx_sasl/src/emqx_sasl_cli.erl +++ b/apps/emqx_sasl/src/emqx_sasl_cli.erl @@ -70,7 +70,7 @@ cli(["scram", "lookup", Username0]) -> salt := Salt, iteration_count := IterationCount}} -> emqx_ctl:print("Username: ~s, Stored Key: ~s, Server Key: ~s, Salt: ~s, Iteration Count: ~p~n", - [Username, StoredKey, ServerKey, Salt, IterationCount]); + [Username, StoredKey, ServerKey, base64:decode(Salt), IterationCount]); {error, not_found} -> emqx_ctl:print("Authentication information not found~n") end; diff --git a/apps/emqx_sasl/test/emqx_sasl_scram_SUITE.erl b/apps/emqx_sasl/test/emqx_sasl_scram_SUITE.erl index 752f19152..de87748f7 100644 --- a/apps/emqx_sasl/test/emqx_sasl_scram_SUITE.erl +++ b/apps/emqx_sasl/test/emqx_sasl_scram_SUITE.erl @@ -77,3 +77,64 @@ t_scram(_) -> {ok, {ok, ServerFinal, #{}}} = emqx_sasl:check(AuthMethod, ClientFinal, Cache), {ok, _} = emqx_sasl:check(AuthMethod, ServerFinal, ClientCache). + +t_proto(_) -> + process_flag(trap_exit, true), + + Username = <<"username">>, + Password = <<"password">>, + Salt = <<"emqx">>, + AuthMethod = <<"SCRAM-SHA-1">>, + + {ok, Client0} = emqtt:start_link([{clean_start, true}, + {proto_ver, v5}, + {enhanced_auth, #{method => AuthMethod, + params => #{username => Username, + password => Password, + salt => Salt}}}, + {connect_timeout, 6000}]), + {error,{not_authorized,#{}}} = emqtt:connect(Client0), + + ok = emqx_sasl_scram:add(Username, Password, Salt), + {ok, Client1} = emqtt:start_link([{clean_start, true}, + {proto_ver, v5}, + {enhanced_auth, #{method => AuthMethod, + params => #{username => Username, + password => Password, + salt => Salt}}}, + {connect_timeout, 6000}]), + {ok, _} = emqtt:connect(Client1), + + timer:sleep(200), + ok = emqtt:reauthentication(Client1, #{params => #{username => Username, + password => Password, + salt => Salt}}), + + timer:sleep(200), + ErrorFun = fun (_State) -> {ok, <<>>, #{}} end, + ok = emqtt:reauthentication(Client1, #{params => #{},function => ErrorFun}), + receive + {disconnected,ReasonCode2,#{}} -> + ?assertEqual(ReasonCode2, 135) + after 500 -> + error("emqx re-authentication failed") + end, + + {ok, Client2} = emqtt:start_link([{clean_start, true}, + {proto_ver, v5}, + {enhanced_auth, #{method => AuthMethod, + params => #{}, + function =>fun (_State) -> {ok, <<>>, #{}} end}}, + {connect_timeout, 6000}]), + {error,{not_authorized,#{}}} = emqtt:connect(Client2), + + receive_msg(), + process_flag(trap_exit, false). + +receive_msg() -> + receive + {'EXIT', Msg} -> + ct:print("==========+~p~n", [Msg]), + receive_msg() + after 200 -> ok + end. diff --git a/apps/emqx_sn/rebar.config b/apps/emqx_sn/rebar.config index 7b1f06cca..7b30a8fd8 100644 --- a/apps/emqx_sn/rebar.config +++ b/apps/emqx_sn/rebar.config @@ -1 +1 @@ -{deps, []}. \ No newline at end of file +{deps, []}. diff --git a/apps/emqx_sn/src/emqx_sn_gateway.erl b/apps/emqx_sn/src/emqx_sn_gateway.erl index aa1f090bb..213aaf385 100644 --- a/apps/emqx_sn/src/emqx_sn_gateway.erl +++ b/apps/emqx_sn/src/emqx_sn_gateway.erl @@ -201,8 +201,7 @@ idle(cast, {incoming, ?SN_PUBLISH_MSG(#mqtt_sn_flags{qos = ?QOS_NEG1, emqx_sn_registry:lookup_topic(Registry, ClientId, TopicId); true -> <> end, - Msg = emqx_message:make({?NEG_QOS_CLIENT_ID, State#state.username}, - ?QOS_0, TopicName, Data), + Msg = emqx_message:make(?NEG_QOS_CLIENT_ID, ?QOS_0, TopicName, Data), (TopicName =/= undefined) andalso emqx_broker:publish(Msg), ?LOG(debug, "Client id=~p receives a publish with QoS=-1 in idle mode!", [ClientId], State), {keep_state_and_data, State#state.idle_timeout}; @@ -942,14 +941,13 @@ do_publish_will(#state{will_msg = #will_msg{payload = undefined}}) -> ok; do_publish_will(#state{will_msg = #will_msg{topic = undefined}}) -> ok; -do_publish_will(#state{channel = Channel, will_msg = WillMsg}) -> +do_publish_will(#state{will_msg = WillMsg, clientid = ClientId}) -> #will_msg{qos = QoS, retain = Retain, topic = Topic, payload = Payload} = WillMsg, Publish = #mqtt_packet{header = #mqtt_packet_header{type = ?PUBLISH, dup = false, qos = QoS, retain = Retain}, variable = #mqtt_packet_publish{topic_name = Topic, packet_id = 1000}, payload = Payload}, - ClientInfo = emqx_channel:info(clientinfo, Channel), - emqx_broker:publish(emqx_packet:to_message(ClientInfo, Publish)), + emqx_broker:publish(emqx_packet:to_message(Publish, ClientId)), ok. do_puback(TopicId, MsgId, ReturnCode, _StateName, diff --git a/apps/emqx_stomp/rebar.config b/apps/emqx_stomp/rebar.config index 7b1f06cca..7b30a8fd8 100644 --- a/apps/emqx_stomp/rebar.config +++ b/apps/emqx_stomp/rebar.config @@ -1 +1 @@ -{deps, []}. \ No newline at end of file +{deps, []}. diff --git a/apps/emqx_telemetry/rebar.config b/apps/emqx_telemetry/rebar.config index 7b1f06cca..7b30a8fd8 100644 --- a/apps/emqx_telemetry/rebar.config +++ b/apps/emqx_telemetry/rebar.config @@ -1 +1 @@ -{deps, []}. \ No newline at end of file +{deps, []}. diff --git a/apps/emqx_web_hook/rebar.config b/apps/emqx_web_hook/rebar.config index 7b1f06cca..7b30a8fd8 100644 --- a/apps/emqx_web_hook/rebar.config +++ b/apps/emqx_web_hook/rebar.config @@ -1 +1 @@ -{deps, []}. \ No newline at end of file +{deps, []}. diff --git a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl index a2cb1dec5..2b0cac618 100644 --- a/apps/emqx_web_hook/src/emqx_web_hook_actions.erl +++ b/apps/emqx_web_hook/src/emqx_web_hook_actions.erl @@ -22,27 +22,40 @@ -define(RESOURCE_TYPE_WEBHOOK, 'web_hook'). -define(RESOURCE_CONFIG_SPEC, #{ - url => #{type => string, + url => #{order => 1, + type => string, format => url, required => true, title => #{en => <<"Request URL">>, zh => <<"请求 URL"/utf8>>}, description => #{en => <<"Request URL">>, zh => <<"请求 URL"/utf8>>}}, - headers => #{type => object, - schema => #{}, - default => #{}, - title => #{en => <<"Request Header">>, - zh => <<"请求头"/utf8>>}, - description => #{en => <<"Request Header">>, - zh => <<"请求头"/utf8>>}}, - method => #{type => string, + method => #{order => 2, + type => string, enum => [<<"PUT">>,<<"POST">>,<<"GET">>,<<"DELETE">>], default => <<"POST">>, title => #{en => <<"Request Method">>, zh => <<"请求方法"/utf8>>}, - description => #{en => <<"Request Method. Note that the payload_template will be discarded in case of GET method">>, - zh => <<"请求方法。注意：当请求方法为 GET 的时候，payload_template 参数会被忽略"/utf8>>}} + description => #{en => <<"Request Method. \n" + "Note that: the Payload Template of Action will be discarded in case of GET method">>, + zh => <<"请求方法。\n" + "注意：当方法为 GET 时，动作中的 '消息内容模板' 参数会被忽略"/utf8>>}}, + content_type => #{order => 3, + type => string, + enum => [<<"application/json">>,<<"text/plain;charset=UTF-8">>], + default => <<"application/json">>, + title => #{en => <<"Content-Type">>, + zh => <<"Content-Type"/utf8>>}, + description => #{en => <<"The Content-Type of HTTP Request">>, + zh => <<"HTTP 请求头中的 Content-Type 字段值"/utf8>>}}, + headers => #{order => 4, + type => object, + schema => #{}, + default => #{}, + title => #{en => <<"Request Header">>, + zh => <<"请求头"/utf8>>}, + description => #{en => <<"The custom HTTP request headers">>, + zh => <<"自定义的 HTTP 请求头列表"/utf8>>}} }). -define(ACTION_PARAM_RESOURCE, #{ @@ -57,18 +70,32 @@ -define(ACTION_DATA_SPEC, #{ '$resource' => ?ACTION_PARAM_RESOURCE, + path => #{order => 1, + type => string, + required => false, + default => <<>>, + title => #{en => <<"Path">>, + zh => <<"Path"/utf8>>}, + description => #{en => <<"A path component, variable interpolation from " + "SQL statement is supported. This value will be " + "concatenated with Request URL.">>, + zh => <<"URL 的路径配置，支持使用 ${} 获取规则输出的字段值。\n" + "例如：${clientid}。该值会与 Request URL 组成一个完整的 URL"/utf8>>} + }, payload_tmpl => #{ - order => 1, + order => 2, type => string, input => textarea, required => false, default => <<"">>, title => #{en => <<"Payload Template">>, zh => <<"消息内容模板"/utf8>>}, - description => #{en => <<"The payload template, variable interpolation is supported. If using empty template (default), then the payload will be all the available vars in JOSN format">>, - zh => <<"消息内容模板，支持变量。若使用空模板（默认），消息内容为 JSON 格式的所有字段"/utf8>>} - } - }). + description => #{en => <<"The payload template, variable interpolation is supported." + "If using empty template (default), then the payload will " + "be all the available vars in JSON format">>, + zh => <<"消息内容模板，支持使用 ${} 获取变量值。" + "默认消息内容为规则输出的所有字段的 JSON 字符串"/utf8>>}} + }). -resource_type(#{name => ?RESOURCE_TYPE_WEBHOOK, create => on_resource_create, @@ -139,11 +166,13 @@ on_resource_destroy(_ResId, _Params) -> %% An action that forwards publish messages to a remote web server. -spec(on_action_create_data_to_webserver(Id::binary(), #{url() := string()}) -> action_fun()). on_action_create_data_to_webserver(_Id, Params) -> - #{url := Url, headers := Headers, method := Method, payload_tmpl := PayloadTmpl} + #{url := Url, headers := Headers, method := Method, content_type := ContentType, payload_tmpl := PayloadTmpl, path := Path} = parse_action_params(Params), PayloadTks = emqx_rule_utils:preproc_tmpl(PayloadTmpl), + PathTks = emqx_rule_utils:preproc_tmpl(Path), fun(Selected, _Envs) -> - http_request(Url, Headers, Method, format_msg(PayloadTks, Selected)) + FullUrl = Url ++ emqx_rule_utils:proc_tmpl(PathTks, Selected), + http_request(FullUrl, Headers, Method, ContentType, format_msg(PayloadTks, Selected)) end. format_msg([], Data) -> @@ -155,15 +184,15 @@ format_msg(Tokens, Data) -> %% Internal functions %%------------------------------------------------------------------------------ -create_req(get, Url, Headers, _) -> +create_req(get, Url, Headers, _, _) -> {(Url), (Headers)}; -create_req(_, Url, Headers, Body) -> - {(Url), (Headers), "application/json", (Body)}. +create_req(_, Url, Headers, ContentType, Body) -> + {(Url), (Headers), binary_to_list(ContentType), (Body)}. -http_request(Url, Headers, Method, Params) -> - logger:debug("[WebHook Action] ~s to ~s, headers: ~p, body: ~p", [Method, Url, Headers, Params]), - case do_http_request(Method, create_req(Method, Url, Headers, Params), +http_request(Url, Headers, Method, ContentType, Params) -> + logger:debug("[WebHook Action] ~s to ~s, headers: ~p, content-type: ~p, body: ~p", [Method, Url, Headers, ContentType, Params]), + case do_http_request(Method, create_req(Method, Url, Headers, ContentType, Params), [{timeout, 5000}], [], 0) of {ok, _} -> ok; {error, Reason} -> @@ -185,7 +214,9 @@ parse_action_params(Params = #{<<"url">> := Url}) -> #{url => str(Url), headers => headers(maps:get(<<"headers">>, Params, undefined)), method => method(maps:get(<<"method">>, Params, <<"POST">>)), - payload_tmpl => maps:get(<<"payload_tmpl">>, Params, <<>>)} + content_type => maps:get(<<"content_type">>, Params, <<"application/json">>), + payload_tmpl => maps:get(<<"payload_tmpl">>, Params, <<>>), + path => maps:get(<<"path">>, Params, <<>>)} catch _:_ -> throw({invalid_params, Params}) end. diff --git a/rebar.config b/rebar.config index 0026218f3..c6de65fde 100644 --- a/rebar.config +++ b/rebar.config @@ -76,7 +76,7 @@ , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}} , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}} , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.1"}}} - , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "v0.4.2"}}} + , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}} , {replayq, {git, "https://github.com/emqx/replayq", {tag, "v0.2.0"}}} , {erlport, {git, "https://github.com/emqx/erlport", {tag, "v1.2.2"}}} , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.3"}}}