diff --git a/Makefile b/Makefile index 33052d6f9..6dc1d0664 100644 --- a/Makefile +++ b/Makefile @@ -152,17 +152,17 @@ define gen-relup-target $1-relup: $(COMMON_DEPS) @$(BUILD) $1 relup endef -ALL_ZIPS = $(REL_PROFILES) -$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-relup-target,$(zt)))) +ALL_TGZS = $(REL_PROFILES) +$(foreach zt,$(ALL_TGZS),$(eval $(call gen-relup-target,$(zt)))) -## zip target is to create a release package .zip with relup -.PHONY: $(REL_PROFILES:%=%-zip) -define gen-zip-target -$1-zip: $1-relup - @$(BUILD) $1 zip +## tgz target is to create a release package .tar.gz with relup +.PHONY: $(REL_PROFILES:%=%-tgz) +define gen-tgz-target +$1-tgz: $1-relup + @$(BUILD) $1 tgz endef -ALL_ZIPS = $(REL_PROFILES) -$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-zip-target,$(zt)))) +ALL_TGZS = $(REL_PROFILES) +$(foreach zt,$(ALL_TGZS),$(eval $(call gen-tgz-target,$(zt)))) ## A pkg target depend on a regular release .PHONY: $(PKG_PROFILES) @@ -185,20 +185,20 @@ define gen-docker-target $1-docker: $(COMMON_DEPS) @$(BUILD) $1 docker endef -ALL_ZIPS = $(REL_PROFILES) -$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) +ALL_TGZS = $(REL_PROFILES) +$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target,$(zt)))) ## emqx-docker-testing ## emqx-enterprise-docker-testing -## is to directly copy a unzipped zip-package to a +## is to directly copy a extracted tgz-package to a ## base image such as ubuntu20.04. Mostly for testing .PHONY: $(REL_PROFILES:%=%-docker-testing) define gen-docker-target-testing $1-docker-testing: $(COMMON_DEPS) @$(BUILD) $1 docker-testing endef -ALL_ZIPS = $(REL_PROFILES) -$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) +ALL_TGZS = $(REL_PROFILES) +$(foreach zt,$(ALL_TGZS),$(eval $(call gen-docker-target-testing,$(zt)))) conf-segs: @scripts/merge-config.escript diff --git a/build b/build index a30d7629d..a0c7b4f85 100755 --- a/build +++ b/build @@ -2,7 +2,7 @@ # This script helps to build release artifacts. # arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg -# arg2: artifact, e.g. rel | relup | zip | pkg +# arg2: artifact, e.g. rel | relup | tgz | pkg set -euo pipefail @@ -79,28 +79,18 @@ make_rel() { fi } -## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup +## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup make_relup() { - local lib_dir="_build/$PROFILE/rel/emqx/lib" - local releases_dir="_build/$PROFILE/rel/emqx/releases" - mkdir -p "$lib_dir" "$releases_dir" + local rel_dir="_build/$PROFILE/rel/emqx" + mkdir -p "${rel_dir}/lib" + mkdir -p "${rel_dir}/releases" local releases=() - if [ -d "$releases_dir" ]; then - while read -r zip; do - local base_vsn - base_vsn="$(echo "$zip" | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+|(-(alpha|beta)\.[0-9]))(-[0-9a-e]{8})?")" - if [ ! -d "$releases_dir/$base_vsn" ]; then - local tmp_dir - tmp_dir="$(mktemp -d -t emqx.XXXXXXX)" - unzip -q "$zip" "emqx/releases/*" -d "$tmp_dir" - unzip -q "$zip" "emqx/lib/*" -d "$tmp_dir" - cp -r -n "$tmp_dir/emqx/releases"/* "$releases_dir" - cp -r -n "$tmp_dir/emqx/lib"/* "$lib_dir" - rm -rf "$tmp_dir" - fi - releases+=( "$base_vsn" ) - done < <("$FIND" _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f) - fi + while read -r tgzfile ; do + local base_vsn + base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)" + tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib + releases+=( "$base_vsn" ) + done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f) if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" return 0 @@ -126,9 +116,9 @@ cp_dyn_libs() { | sort -u) } -## make_zip turns .tar.gz into a .zip with a slightly different name. +## Re-pack the relx assembled .tar.gz to EMQ X's package naming scheme ## It assumes the .tar.gz has been built -- relies on Makefile dependency -make_zip() { +make_tgz() { # build the tarball again to ensure relup is included make_rel @@ -142,14 +132,14 @@ make_zip() { if [ ! -f "$tarball" ]; then log "ERROR: $tarball is not found" fi - local zipball - zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" + local target + target="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" tar zxf "${tarball}" -C "${tard}/emqx" - ## try to be portable for zip packages. + ## try to be portable for tar.gz packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt cp_dyn_libs "${tard}/emqx" - (cd "${tard}" && zip -qr - emqx) > "${zipball}" - log "Zip package successfully created: ${zipball}" + (cd "${tard}" && tar -cz emqx) > "${target}" + log "Tarball successfully repacked: ${target}" } ## This function builds the default docker image based on alpine:3.14 (by default) @@ -166,7 +156,7 @@ make_docker() { } ## This function accepts any base docker image, -## a emqx zip-image, and a image tag (for the image to be built), +## a emqx tgz-image, and a image tag (for the image to be built), ## to build a docker image which runs EMQ X ## ## Export below variables to quickly build an image @@ -174,7 +164,7 @@ make_docker() { ## Name Default Example ## --------------------------------------------------------------------- ## EMQX_BASE_IMAGE current os centos:7 -## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip +## EMQX_TGZ_packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.tar.gz ## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag ## make_docker_testing() { @@ -193,17 +183,17 @@ make_docker_testing() { esac fi EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" - local defaultzip - defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" - local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" - if [ ! -f "$zip" ]; then - log "ERROR: $zip not built?" + local default_tgz + default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz" + local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}" + if [ ! -f "$tgz" ]; then + log "ERROR: $tgz not built?" exit 1 fi set -x docker build \ --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ - --build-arg EMQX_ZIP_PACKAGE="${zip}" \ + --build-arg EMQX_TGZ_PACKAGE="${tgz}" \ --tag "$EMQX_IMAGE_TAG" \ -f "${DOCKERFILE_TESTING}" . } @@ -220,8 +210,8 @@ case "$ARTIFACT" in relup) make_relup ;; - zip) - make_zip + tgz) + make_tgz ;; pkg) if [ -z "${PKGERDIR:-}" ]; then diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 7c49ce909..1ed0fd56d 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -8,7 +8,7 @@ ## i.e. will not work if docker command has to be executed with sudo ## example: -## ./scripts/buildx.sh --profile emqx --pkgtype zip --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 +## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 --builder ghcr.io/emqx/emqx-builder/4.4-4:24.1.5-3-debian10 set -euo pipefail @@ -16,7 +16,8 @@ help() { echo echo "-h|--help: To display this usage information" echo "--profile : EMQ X profile to build, e.g. emqx, emqx-edge" - echo "--pkgtype zip|pkg: Specify which package to build, zip for .zip and pkg for .rpm or .deb" + echo "--pkgtype tgz|pkg: Specify which package to build, tgz for .tar.gz" + echo " and pkg for .rpm or .deb" echo "--arch amd64|arm64: Target arch to build the EMQ X package for" echo "--src_dir : EMQ X source ode in this dir, default to PWD" echo "--builder : Builder image to pull" @@ -62,8 +63,8 @@ if [ -z "${PROFILE:-}" ] || [ -z "${PKGTYPE:-}" ] || [ -z "${BUILDER:-}" ] || [ exit 1 fi -if [ "$PKGTYPE" != 'zip' ] && [ "$PKGTYPE" != 'pkg' ]; then - echo "Bad --pkgtype option, should be zip or pkg" +if [ "$PKGTYPE" != 'tgz' ] && [ "$PKGTYPE" != 'pkg' ]; then + echo "Bad --pkgtype option, should be tgz or pkg" exit 1 fi diff --git a/scripts/update_appup.escript b/scripts/update_appup.escript index 262ff906f..a713ca8b0 100755 --- a/scripts/update_appup.escript +++ b/scripts/update_appup.escript @@ -32,8 +32,8 @@ Options: --make-command A command used to assemble the release --release-dir Release directory --src-dirs Directories where source code is found. Defaults to '{src,apps,lib-*}/**/' - --binary-rel-url Binary release URL pattern. %TAG% variable is substituted with the release tag. - E.g. \"https://github.com/emqx/emqx/releases/download/v%TAG%/emqx-centos7-%TAG%-amd64.zip\" + --binary-rel-url Binary release URL pattern. %VSN% variable is substituted with the version in release tag. + E.g. \"https://github.com/emqx/emqx/releases/download/v%VSN%/emqx-%VSN%-otp-24.1.5-3-centos7-amd64.tar.gz\" ". -record(app, @@ -172,8 +172,8 @@ download_prev_release(Tag, #{binary_rel_url := {ok, URL0}, clone_url := Repo}) - Dir = filename:basename(Repo, ".git") ++ [$-|Tag], Filename = filename:join(BaseDir, Dir), Script = "mkdir -p ${OUTFILE} && - wget -c -O ${OUTFILE}.zip ${URL} && - unzip -n -d ${OUTFILE} ${OUTFILE}.zip", + wget -c -O ${OUTFILE}.tar.gz ${URL} && + tar -zxf ${OUTFILE} ${OUTFILE}.tar.gz", Env = [{"TAG", Tag}, {"OUTFILE", Filename}, {"URL", URL}], bash(Script, Env), {ok, Filename}.