diff --git a/.ci/build_packages/tests.sh b/.ci/build_packages/tests.sh index c4b752da1..6eaaaa15e 100755 --- a/.ci/build_packages/tests.sh +++ b/.ci/build_packages/tests.sh @@ -1,11 +1,52 @@ -#!/bin/bash -set -x -e -u +#!/usr/bin/env bash + +set -euo pipefail +set -x + +if [ -z "${1:-}" ]; then + echo "Usage $0 zip|pkg" + exit 1 +fi + +case "${2:-}" in + zip|pkg|elixirpkg) + true + ;; + *) + echo "Usage $0 zip|pkg" + exit 1 + ;; +esac + +PACKAGE_NAME="${1}" +PACKAGE_TYPE="${2}" + export DEBUG=1 export CODE_PATH=${CODE_PATH:-"/emqx"} export EMQX_NAME=${EMQX_NAME:-"emqx"} export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}" export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base" +if [ "$PACKAGE_TYPE" = 'zip' ]; then + PKG_SUFFIX="zip" +else + SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)" + case "${SYSTEM:-}" in + ubuntu*|debian*|raspbian*) + PKG_SUFFIX='deb' + ;; + *) + PKG_SUFFIX='rpm' + ;; + esac +fi +PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}" + +PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}" +if ! [ -f "$PACKAGE_FILE" ]; then + echo "$PACKAGE_FILE is not a file" +fi + case "$(uname -m)" in x86_64) ARCH='amd64' @@ -30,11 +71,9 @@ emqx_prepare(){ emqx_test(){ cd "${PACKAGE_PATH}" - - for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do - case ${var##*.} in + local packagename="${PACKAGE_FILE_NAME}" + case "$PKG_SUFFIX" in "zip") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip) unzip -q "${PACKAGE_PATH}/${packagename}" export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60 export EMQX_MQTT__MAX_TOPIC_ALIAS=10 @@ -42,7 +81,6 @@ emqx_test(){ export EMQX_LOG__FILE_HANDLERS__DEFAULT__LEVEL=debug if [[ $(arch) == *arm* || $(arch) == aarch64 ]]; then export EMQX_LISTENERS__QUIC__DEFAULT__ENABLED=false - export WAIT_FOR_ERLANG_STOP=120 fi # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins @@ -72,7 +110,6 @@ emqx_test(){ rm -rf "${PACKAGE_PATH}"/emqx ;; "deb") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb) dpkg -i "${PACKAGE_PATH}/${packagename}" if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ] then @@ -99,15 +136,8 @@ emqx_test(){ fi ;; "rpm") - packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm) - - if [[ "${ARCH}" == "amd64" && $(rpm -E '%{rhel}') == 7 ]] ; - then - # EMQX OTP requires openssl11 to have TLS1.3 support - yum install -y openssl11; - fi - rpm -ivh "${PACKAGE_PATH}/${packagename}" - if ! rpm -q emqx | grep -q emqx; then + yum install -y "${PACKAGE_PATH}/${packagename}" + if ! rpm -q "${EMQX_NAME}" | grep -q "${EMQX_NAME}"; then echo "package install error" exit 1 fi @@ -124,7 +154,6 @@ emqx_test(){ ;; esac - done } run_test(){ @@ -142,7 +171,6 @@ EOF ## for ARM, due to CI env issue, skip start of quic listener for the moment [[ $(arch) == *arm* || $(arch) == aarch64 ]] && tee -a "$emqx_env_vars" <> $GITHUB_ENV + echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + echo "EMQX_PKG_NAME=${{ matrix.profile }}-$(./pkg-vsn.sh)-otp${{ matrix.otp }}-${{ matrix.os }}-amd64" >> $GITHUB_ENV + - name: build and test zip package + run: | + make ${EMQX_NAME}-zip + .ci/build_packages/tests.sh "$EMQX_PKG_NAME" zip + - name: build and test deb/rpm packages + run: | + make ${EMQX_NAME}-pkg + .ci/build_packages/tests.sh "$EMQX_PKG_NAME" pkg - uses: actions/upload-artifact@v2 with: name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} path: _packages/**/*.zip - mac: if: contains(github.event.pull_request.labels.*.name, 'Mac') strategy: diff --git a/Makefile b/Makefile index 9cff71584..4fed25635 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ SCRIPTS = $(CURDIR)/scripts export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 export EMQX_DEFAULT_RUNNER = alpine:3.14 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) +export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) export EMQX_DASHBOARD_VERSION ?= v0.10.0 export DOCKERFILE := deploy/docker/Dockerfile @@ -34,6 +35,18 @@ ensure-rebar3: @$(SCRIPTS)/fail-on-old-otp-version.escript @$(SCRIPTS)/ensure-rebar3.sh $(REBAR_VERSION) +.PHONY: ensure-hex +ensure-hex: + @mix local.hex --if-missing --force + +.PHONY: ensure-mix-rebar3 +ensure-mix-rebar3: $(REBAR) + @mix local.rebar rebar3 $(CURDIR)/rebar3 --if-missing --force + +.PHONY: ensure-mix-rebar +ensure-mix-rebar: $(REBAR) + @mix local.rebar --if-missing --force + $(REBAR): ensure-rebar3 .PHONY: get-dashboard @@ -89,7 +102,7 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) -$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs +$(REL_PROFILES:%=%): $(COMMON_DEPS) @$(REBAR) as $(@) do release ## Not calling rebar3 clean because @@ -133,6 +146,7 @@ dialyzer: $(REBAR) @$(REBAR) as check dialyzer COMMON_DEPS := $(REBAR) get-dashboard conf-segs +ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar ## rel target is to create release package without relup .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel) @@ -195,3 +209,18 @@ $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) conf-segs: @scripts/merge-config.escript + +## elixir target is to create release packages using Elixir's Mix +.PHONY: $(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir) +$(REL_PROFILES:%=%-elixir) $(PKG_PROFILES:%=%-elixir): $(COMMON_DEPS) $(ELIXIR_COMMON_DEPS) + @$(BUILD) $(subst -elixir,,$(@)) elixir + +.PHONY: $(REL_PROFILES:%=%-elixirpkg) +define gen-elixirpkg-target +# the Elixir places the tar in a different path than Rebar3 +$1-elixirpkg: $1-pkg-elixir + @env TAR_PKG_DIR=_build/prod \ + IS_ELIXIR=yes \ + $(BUILD) $1 pkg +endef +$(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixirpkg-target,$(pt)))) diff --git a/bin/emqx b/bin/emqx index bde1f0b51..1d0bfa4bf 100755 --- a/bin/emqx +++ b/bin/emqx @@ -578,15 +578,14 @@ case "${COMMAND}" in "$BINDIR/run_erl" -daemon "$PIPE_DIR" "$RUNNER_LOG_DIR" \ "$(relx_start_command)" - WAIT_TIME=${WAIT_FOR_ERLANG:-15} + WAIT_TIME=${EMQX_WAIT_FOR_START:-120} if wait_for "$WAIT_TIME" 'relx_nodetool' 'ping'; then echo "$EMQX_DESCRIPTION $REL_VSN is started successfully!" exit 0 else - echo "$EMQX_DESCRIPTION $REL_VSN failed to start within ${WAIT_TIME} seconds," - echo "see the output of '$0 console' for more information." - echo "If you want to wait longer, set the environment variable" - echo "WAIT_FOR_ERLANG to the number of seconds to wait." + echo "$EMQX_DESCRIPTION $REL_VSN failed to start in ${WAIT_TIME} seconds." + echo "Please find more information in erlang.log.N" + echo "Or run 'DEBUG=1 $0 console' to have logs printed to console." exit 1 fi ;; @@ -598,13 +597,13 @@ case "${COMMAND}" in echoerr "Graceful shutdown failed PID=[$PID]" exit 1 fi - WAIT_TIME="${WAIT_FOR_ERLANG_STOP:-60}" + WAIT_TIME="${EMQX_WAIT_FOR_STOP:-120}" if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then msg="dangling after ${WAIT_TIME} seconds" # also log to syslog logger -t "${REL_NAME}[${PID}]" "STOP: $msg" # log to user console - echoerr "stop failed, $msg" + echoerr "Stop failed, $msg" echo "ERROR: $PID is still around" ps -p "$PID" exit 1 diff --git a/build b/build index 15fcee226..e239f0ad5 100755 --- a/build +++ b/build @@ -6,6 +6,11 @@ set -euo pipefail +DEBUG="${DEBUG:-0}" +if [ "$DEBUG" -eq 1 ]; then + set -x +fi + PROFILE="$1" ARTIFACT="$2" @@ -202,6 +207,53 @@ make_docker_testing() { -f "${DOCKERFILE_TESTING}" . } +# used to control the Elixir Mix Release output +# see docstring in `mix.exs` +export_release_vars() { + local profile="$1" + case "$profile" in + emqx) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=no + ;; + emqx-edge) + export EMQX_RLEASE_TYPE=edge \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=no + ;; + emqx-enterprise) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=bin \ + EMQX_EDITION_TYPE=enterprise \ + ELIXIR_MAKE_TAR=no + ;; + emqx-pkg) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=yes + ;; + emqx-edge-pkg) + export EMQX_RLEASE_TYPE=edge \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=community \ + ELIXIR_MAKE_TAR=yes + ;; + emqx-enterprise-pkg) + export EMQX_RLEASE_TYPE=cloud \ + EMQX_PACKAGE_TYPE=pkg \ + EMQX_EDITION_TYPE=enterprise \ + ELIXIR_MAKE_TAR=yes + ;; + *) + echo Invalid profile "$profile" + exit 1 + esac +} + log "building artifact=$ARTIFACT for profile=$PROFILE" case "$ARTIFACT" in @@ -223,7 +275,10 @@ case "$ARTIFACT" in exit 0 fi make -C "deploy/packages/${PKGERDIR}" clean - EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}" + env EMQX_REL="$(pwd)" \ + EMQX_BUILD="${PROFILE}" \ + SYSTEM="${SYSTEM}" \ + make -C "deploy/packages/${PKGERDIR}" ;; docker) make_docker @@ -231,6 +286,10 @@ case "$ARTIFACT" in docker-testing) make_docker_testing ;; + elixir) + export_release_vars "$PROFILE" + env MIX_ENV=prod mix release --overwrite + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 2cb3679ee..3b51a1442 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -6,9 +6,16 @@ BUILT := $(SRCDIR)/BUILT EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +ifeq ($(IS_ELIXIR), yes) + ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) +else + ELIXIR_PKG_VSN := +endif + +TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx +TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz +SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)$(ELIXIR_PKG_VSN)-$(SYSTEM)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index acee8b51c..fbb5f899b 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -16,9 +16,16 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) -TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) -SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +ifeq ($(IS_ELIXIR), yes) + ELIXIR_PKG_VSN := -elixir$(ELIXIR_VSN) +else + ELIXIR_PKG_VSN := +endif + +TAR_PKG_DIR ?= _build/$(EMQX_BUILD)/rel/emqx +TAR_PKG := $(EMQX_REL)/$(TAR_PKG_DIR)/emqx-$(PKG_VSN).tar.gz +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)$(ELIXIR_PKG_VSN)-$(SYSTEM)-$(ARCH) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -55,4 +62,3 @@ $(BUILT): clean: rm -rf $(SRCDIR) - diff --git a/mix.exs b/mix.exs index a32512f4e..b0ff36ce7 100644 --- a/mix.exs +++ b/mix.exs @@ -123,7 +123,7 @@ defmodule EMQXUmbrella.MixProject do steps = if System.get_env("ELIXIR_MAKE_TAR") == "yes" do - base_steps ++ [:tar] + base_steps ++ [&prepare_tar_overlays/1, :tar] else base_steps end @@ -232,6 +232,10 @@ defmodule EMQXUmbrella.MixProject do } end + ############################################################################# + # Custom Steps + ############################################################################# + defp copy_files(release, release_type, package_type, edition_type) do overwrite? = Keyword.get(release.options, :overwrite, false) @@ -407,6 +411,26 @@ defmodule EMQXUmbrella.MixProject do release end + # The `:tar` built-in step in Mix Release does not currently add the + # `etc` directory into the resulting tarball. The workaround is to + # add those to the `:overlays` key before running `:tar`. + # See: https://hexdocs.pm/mix/1.13.1/Mix.Release.html#__struct__/0 + defp prepare_tar_overlays(release) do + Enum.each( + ["mnesia", "configs", "patches", "scripts"], + fn dir -> + path = Path.join([release.path, "data", dir]) + File.mkdir_p!(path) + end + ) + + Map.update!(release, :overlays, &["etc", "data" | &1]) + end + + ############################################################################# + # Helper functions + ############################################################################# + defp template_vars(release, release_type, :bin = _package_type, edition_type) do [ platform_bin_dir: "bin", diff --git a/scripts/buildx.sh b/scripts/buildx.sh new file mode 100755 index 000000000..02227957a --- /dev/null +++ b/scripts/buildx.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +## This script helps to run docker buildx to build cross-arch/platform packages (linux only) +## It mounts (not copy) host directory to a cross-arch/platform builder container +## Make sure the source dir (specified by --src_dir option) is clean before running this script + +## NOTE: it requires $USER in docker group +## i.e. will not work if docker command has to be executed with sudo + +## example: +## ./scripts/buildx.sh --profile emqx --pkgtype zip --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 + +set -euo pipefail + +help() { + echo + echo "-h|--help: To display this usage information" + echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" + echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb" + echo "--arch amd64|arm64: Target arch to build the EMQ X package for" + echo "--src_dir : EMQ X source ode in this dir, default to PWD" + echo "--builder : Builder image to pull" + echo " E.g. ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10" +} + +while [ "$#" -gt 0 ]; do + case $1 in + -h|--help) + help + exit 0 + ;; + --src_dir) + SRC_DIR="$2" + shift 2 + ;; + --profile) + PROFILE="$2" + shift 2 + ;; + --pkgtype) + PKGTYPE="$2" + shift 2 + ;; + --builder) + BUILDER="$2" + shift 2 + ;; + --arch) + ARCH="$2" + shift 2 + ;; + *) + echo "WARN: Unknown arg (ignored): $1" + shift + continue + ;; + esac +done + +if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ -z "${ARCH:-}" ]; then + help + exit 1 +fi + +case "$PKGTYPE" in + zip|pkg|elixirpkg) + true + ;; + *) + echo "Bad --pkgtype option, should be zip or pkg" + exit 1 + ;; +esac + +cd "${SRC_DIR:-.}" + +get_otp_vsn () { + docker run -i --rm \ + -v "$(pwd)":/emqx \ + --workdir /emqx \ + --platform="linux/$ARCH" \ + "$BUILDER" \ + bash -euc "./scripts/get-otp-vsn.sh" +} + +get_elixir_vsn () { + docker run -i --rm \ + -v "$(pwd)":/emqx \ + --workdir /emqx \ + --platform="linux/$ARCH" \ + "$BUILDER" \ + bash -euc "./scripts/get-elixir-vsn.sh" +} + +PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" +OTP_VSN=$(get_otp_vsn) +ELIXIR_VSN=$(get_otp_vsn) +SYSTEM=$(echo "$BUILDER" | cut -d ':' -f2) +SYSTEM=${SYSTEM#$OTP_VSN-} +SYSTEM=${SYSTEM#$ELIXIR_VSN-} + +if [ "$PKGTYPE" = "elixirpkg" ] +then + PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-elixir${ELIXIR_VSN}-${SYSTEM}-${ARCH}" +else + PKG_NAME="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" +fi + +docker info +docker run --rm --privileged tonistiigi/binfmt:latest --install "${ARCH}" +docker run -i --rm \ + -v "$(pwd)":/emqx \ + --workdir /emqx \ + --platform="linux/$ARCH" \ + -e EMQX_NAME="$PROFILE" \ + "$BUILDER" \ + bash -euc "make ${PROFILE}-${PKGTYPE} && .ci/build_packages/tests.sh $PKG_NAME $PKGTYPE" diff --git a/scripts/get-elixir-vsn.sh b/scripts/get-elixir-vsn.sh new file mode 100755 index 000000000..c74af4bf6 --- /dev/null +++ b/scripts/get-elixir-vsn.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -euo pipefail + +elixir -e "System.version() |> IO.puts()" diff --git a/scripts/start-two-nodes-in-docker.sh b/scripts/start-two-nodes-in-docker.sh index e0526c26f..45942076f 100755 --- a/scripts/start-two-nodes-in-docker.sh +++ b/scripts/start-two-nodes-in-docker.sh @@ -37,7 +37,6 @@ docker run -d -t --restart=always --name "$NODE1" \ --net "$NET" \ -e EMQX_NODE_NAME="emqx@$NODE1" \ -e EMQX_NODE_COOKIE="$COOKIE" \ - -e WAIT_FOR_ERLANG=60 \ -p 18083:18083 \ -v "$PROJ_DIR"/_build/emqx/rel/emqx:/built \ "$IMAGE" sh -c 'cp -r /built /emqx && /emqx/bin/emqx console' @@ -46,7 +45,6 @@ docker run -d -t --restart=always --name "$NODE2" \ --net "$NET" \ -e EMQX_NODE_NAME="emqx@$NODE2" \ -e EMQX_NODE_COOKIE="$COOKIE" \ - -e WAIT_FOR_ERLANG=60 \ -p 18084:18083 \ -v "$PROJ_DIR"/_build/emqx/rel/emqx:/built \ "$IMAGE" sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'