diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml index 1b61b0278..4c53ec3ae 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml @@ -1,5 +1,4 @@ x-default-emqx: &default-emqx - image: $TARGET:$EMQX_TAG env_file: - conf.cluster.env healthcheck: @@ -15,8 +14,8 @@ services: restart: on-failure environment: - "EMQX_HOST=node1.emqx.io" - - "EMQX_CLUSTER__DB_BACKEND=rlog" - - "EMQX_CLUSTER__RLOG__ROLE=core" + - "EMQX_DB__BACKEND=rlog" + - "EMQX_DB__ROLE=core" - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" - "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false" @@ -29,9 +28,9 @@ services: restart: on-failure environment: - "EMQX_HOST=node2.emqx.io" - - "EMQX_CLUSTER__DB_BACKEND=rlog" - - "EMQX_CLUSTER__RLOG__ROLE=replicant" - - "EMQX_CLUSTER__RLOG__CORE_NODES=emqx@node1.emqx.io" + - "EMQX_DB__BACKEND=rlog" + - "EMQX_DB__ROLE=replicant" + - "EMQX_DB__CORE_NODES=emqx@node1.emqx.io" - "EMQX_CLUSTER__STATIC__SEEDS=[emqx@node1.emqx.io]" - "EMQX_LISTENERS__TCP__DEFAULT__PROXY_PROTOCOL=false" - "EMQX_LISTENERS__WS__DEFAULT__PROXY_PROTOCOL=false" diff --git a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml index b2635ecfe..65bf4faf8 100644 --- a/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml +++ b/.ci/docker-compose-file/docker-compose-emqx-cluster.yaml @@ -1,7 +1,7 @@ version: '3.9' x-default-emqx: &default-emqx - image: $TARGET:$EMQX_TAG + image: ${_EMQX_DOCKER_IMAGE_TAG} env_file: - conf.cluster.env healthcheck: diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 85b1a06be..2f0137428 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -3,7 +3,7 @@ version: '3.9' services: erlang23: container_name: erlang23 - image: ghcr.io/emqx/emqx-builder/5.0:23.2.7.2-emqx-2-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-2:23.3.4.9-3-ubuntu20.04 env_file: - conf.env environment: @@ -23,7 +23,7 @@ services: erlang24: container_name: erlang24 - image: ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04 env_file: - conf.env environment: diff --git a/.ci/docker-compose-file/scripts/run-emqx.sh b/.ci/docker-compose-file/scripts/run-emqx.sh index 1465cb655..4f95db3ce 100755 --- a/.ci/docker-compose-file/scripts/run-emqx.sh +++ b/.ci/docker-compose-file/scripts/run-emqx.sh @@ -1,7 +1,11 @@ #!/bin/bash set -euxo pipefail -if [ "$EMQX_TEST_DB_BACKEND" = "rlog" ] +# _EMQX_DOCKER_IMAGE_TAG is shared with docker-compose file +export _EMQX_DOCKER_IMAGE_TAG="$1" +_EMQX_TEST_DB_BACKEND="${2:-${_EMQX_TEST_DB_BACKEND:-mnesia}}" + +if [ "$_EMQX_TEST_DB_BACKEND" = "rlog" ] then CLUSTER_OVERRIDES="-f .ci/docker-compose-file/docker-compose-emqx-cluster-rlog.override.yaml" else diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 4704e189b..f8f4eb9dc 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -11,92 +11,69 @@ on: types: - published workflow_dispatch: + inputs: + which_branch: + required: false jobs: prepare: - strategy: - matrix: - otp: - - "23.2.7.2-emqx-2" - - "24.1.1-emqx-1" - runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" + # prepare source with any OTP version, no need for a matrix + container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04" outputs: - profiles: ${{ steps.set_profile.outputs.profiles }} - old_vsns: ${{ steps.set_profile.outputs.old_vsns }} + old_vsns: ${{ steps.find_old_versons.outputs.old_vsns }} steps: - uses: actions/checkout@v2 with: + ref: ${{ github.event.inputs.which_branch }} path: source fetch-depth: 0 - - name: set profile - id: set_profile + - name: find old versions + id: find_old_versons shell: bash working-directory: source run: | vsn="$(./pkg-vsn.sh)" pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - if make emqx-ee --dry-run > /dev/null 2>&1; then - old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")" - echo "::set-output name=old_vsns::$old_vsns" - echo "::set-output name=profiles::[\"emqx-ee\"]" - else - old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" - echo "::set-output name=old_vsns::$old_vsns" - echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]" - fi - - name: get otp version - id: get_otp_version + old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" + echo "::set-output name=old_vsns::$old_vsns" + - name: get_all_deps run: | - otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)" - echo "::set-output name=otp::$otp" - - name: set get token - if: endsWith(github.repository, 'enterprise') - run: | - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - - name: get deps - working-directory: source - run: | - make ensure-rebar3 - ./rebar3 as default get-deps - rm -rf rebar.lock - - name: gen zip file - run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]* + make -C source deps-all + zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v2 with: - name: source-${{ steps.get_otp_version.outputs.otp }} - path: source-${{ steps.get_otp_version.outputs.otp }}.zip + name: source + path: source.zip windows: runs-on: windows-2019 needs: prepare - if: endsWith(github.repository, 'emqx') strategy: fail-fast: false matrix: - profile: ${{fromJSON(needs.prepare.outputs.profiles)}} - exclude: - - profile: emqx-edge + profile: # only CE for windows + - emqx + otp: + - 23.2 steps: - uses: actions/download-artifact@v2 with: - name: source-23.2.7.2-emqx-2 + name: source path: . - name: unzip source code - run: Expand-Archive -Path source-23.2.7.2-emqx-2.zip -DestinationPath ./ + run: Expand-Archive -Path source.zip -DestinationPath ./ - uses: ilammy/msvc-dev-cmd@v1 - uses: gleam-lang/setup-erlang@v1.1.2 id: install_erlang ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows with: - otp-version: 23.2 + otp-version: ${{ matrix.otp }} - name: build env: PYTHON: python @@ -108,10 +85,10 @@ jobs: $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" ) if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") { $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+" - $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip" + $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip" } else { - $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip" + $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip" } ## We do not build/release bcrypt and quic for windows package Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/ @@ -140,34 +117,32 @@ jobs: - uses: actions/upload-artifact@v1 if: startsWith(github.ref, 'refs/tags/') with: - name: ${{ matrix.profile }}-23.2.7.2-emqx-2 + name: ${{ matrix.profile }} path: source/_packages/${{ matrix.profile }}/. mac: - needs: prepare - strategy: fail-fast: false matrix: - profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + profile: # no EDGE for mac + - emqx + - emqx-ee + otp: + - 24.1.5-2 macos: - macos-11 - macos-10.15 - otp: - - 24.1.1-emqx-1 exclude: - profile: emqx-edge - runs-on: ${{ matrix.macos }} - steps: - uses: actions/download-artifact@v2 with: - name: source-${{ matrix.otp }} + name: source path: . - name: unzip source code - run: unzip -q source-${{ matrix.otp }}.zip + run: unzip -q source.zip - name: prepare run: | brew update @@ -182,8 +157,12 @@ jobs: - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 + env: + KERL_BUILD_BACKEND: git + OTP_GITHUB_URL: https://github.com/emqx/otp run: | - kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }} + kerl update releases + kerl build ${{ matrix.otp }} kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} - name: build working-directory: source @@ -191,11 +170,12 @@ jobs: . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 + rm -rf _build/${{ matrix.profile }}/lib make ${{ matrix.profile }}-zip - name: test working-directory: source run: | - pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip | head) + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) unzip -q $pkg_name # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 @@ -230,7 +210,15 @@ jobs: strategy: fail-fast: false matrix: - profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + profile: ## all editions for linux + - emqx-edge + - emqx + - emqx-ee + otp: + - 24.1.5-2 # we test with OTP 23, but only build package on OTP 24 versions + arch: + - amd64 + - arm64 os: - ubuntu20.04 - ubuntu18.04 @@ -240,18 +228,9 @@ jobs: # - opensuse - centos8 - centos7 - - centos6 - raspbian10 # - raspbian9 - arch: - - amd64 - - arm64 - otp: - - 23.2.7.2-emqx-2 - - 24.1.1-emqx-1 exclude: - - os: centos6 - arch: arm64 - os: raspbian9 arch: amd64 - os: raspbian10 @@ -277,12 +256,13 @@ jobs: platforms: all - uses: actions/download-artifact@v2 with: - name: source-${{ matrix.otp }} + name: source path: . - name: unzip source code - run: unzip -q source-${{ matrix.otp }}.zip + run: unzip -q source.zip - name: downloads old emqx zip packages env: + OTP_VSN: ${{ matrix.otp }} PROFILE: ${{ matrix.profile }} ARCH: ${{ matrix.arch }} SYSTEM: ${{ matrix.os }} @@ -302,10 +282,11 @@ jobs: cd _upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for tag in ${old_vsns[@]}; do - if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256 - echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1 + package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}" + if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256 + echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1 fi done - name: build emqx packages @@ -320,7 +301,7 @@ jobs: -v $(pwd):/emqx \ --workdir /emqx \ --platform linux/$ARCH \ - ghcr.io/emqx/emqx-builder/5.0:$OTP-$SYSTEM \ + ghcr.io/emqx/emqx-builder/5.0-2:$OTP-$SYSTEM \ bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \ make $PROFILE-pkg || cat rebar3.crashdump; \ EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh" @@ -349,17 +330,21 @@ jobs: strategy: fail-fast: false matrix: - profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + profile: # all editions for docker + - emqx-edge + - emqx + - emqx-ee + # NOTE: for docker, only support latest otp version, not a matrix otp: - - 24.1.1-emqx-1 + - 24.1.5-2 # update to latest steps: - uses: actions/download-artifact@v2 with: - name: source-${{ matrix.otp }} + name: source path: . - name: unzip source code - run: unzip -q source-${{ matrix.otp }}.zip + run: unzip -q source.zip - uses: docker/setup-buildx-action@v1 - uses: docker/setup-qemu-action@v1 with: @@ -376,7 +361,8 @@ jobs: type=ref,event=pr type=ref,event=tag type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} + labels: + org.opencontainers.image.otp.version=${{ matrix.otp }} - uses: docker/login-action@v1 if: github.event_name == 'release' with: @@ -384,32 +370,26 @@ jobs: password: ${{ secrets.DOCKER_HUB_TOKEN }} - uses: docker/build-push-action@v2 with: - push: ${{ github.event_name == 'release' }} + push: ${{ github.event_name == 'release' && !github.event.release.prerelease }} pull: true no-cache: true platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-alpine3.14 + BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-alpine3.14 RUN_FROM=alpine:3.14 EMQX_NAME=${{ matrix.profile }} file: source/deploy/docker/Dockerfile context: source delete-artifact: - runs-on: ubuntu-20.04 - strategy: - matrix: - otp: - - 23.2.7.2-emqx-2 - - 24.1.1-emqx-1 needs: [prepare, mac, linux, docker] steps: - uses: geekyeggo/delete-artifact@v1 with: - name: source-${{ matrix.otp }} + name: source upload: runs-on: ubuntu-20.04 @@ -420,9 +400,12 @@ jobs: strategy: matrix: - profile: ${{fromJSON(needs.prepare.outputs.profiles)}} + profile: + - emqx-edge + - emqx + - emqx-ee otp: - - 24.1.1-emqx-1 + - 24.1.5-2 steps: - uses: actions/checkout@v2 @@ -461,17 +444,11 @@ jobs: aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }} aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*" - uses: Rory-Z/upload-release-asset@v1 - if: github.event_name == 'release' && matrix.profile != 'emqx-ee' + if: github.event_name == 'release' with: repo: emqx path: "_packages/${{ matrix.profile }}/emqx-*" token: ${{ github.token }} - - uses: Rory-Z/upload-release-asset@v1 - if: github.event_name == 'release' && matrix.profile == 'emqx-ee' - with: - repo: emqx-enterprise - path: "_packages/${{ matrix.profile }}/emqx-*" - token: ${{ github.token }} - name: update to emqx.io if: github.event_name == 'release' run: | @@ -484,32 +461,28 @@ jobs: -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \ ${{ secrets.EMQX_IO_RELEASE_API }} - name: update repo.emqx.io - if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee' + if: github.event_name == 'release' run: | + if [ "${{ matrix. profile }}" = 'emqx-ee' ]; then + BOOL_FLAG_NAME="emqx_ee" + else + BOOL_FLAG_NAME="emqx_ce" + fi curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \ - "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - - name: update repo.emqx.io - if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' - run: | - curl --silent --show-error \ - -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \ + -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"${BOOL_FLAG_NAME}\": \"true\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches" - name: update homebrew packages - if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx' + if: github.event_name == 'release' && matrix.profile == 'emqx' run: | if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then curl --silent --show-error \ -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \ -H "Accept: application/vnd.github.v3+json" \ -X POST \ - -d "{\"ref\":\"v1.0.3\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ + -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \ "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches" fi - uses: geekyeggo/delete-artifact@v1 diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 67e813747..46eb7aef7 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -4,7 +4,6 @@ concurrency: group: slim-${{ github.event_name }}-${{ github.ref }} cancel-in-progress: true - on: push: tags: @@ -14,48 +13,37 @@ on: workflow_dispatch: jobs: - build: + linux: runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: + profile: + - emqx-edge + - emqx + - emqx-ee otp: - - 24.1.1-emqx-1 + - 24.1.5-2 os: - ubuntu20.04 - centos7 - container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-${{ matrix.os }}" steps: - uses: actions/checkout@v1 - - name: prepare - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token - echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV - else - echo "EMQX_NAME=emqx" >> $GITHUB_ENV - fi - - name: build zip packages - run: make ${EMQX_NAME}-zip + - name: build zip package + run: make ${{ matrix.profile }}-zip - name: build deb/rpm packages - run: make ${EMQX_NAME}-pkg - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: rebar3.crashdump - path: ./rebar3.crashdump + run: make ${{ matrix.profile }}-pkg - name: packages test run: | export CODE_PATH=$GITHUB_WORKSPACE - .ci/build_packages/tests.sh + EMQX_NAME=${{ matrix.profile }} .ci/build_packages/tests.sh - uses: actions/upload-artifact@v2 with: - name: ${{ matrix.os }} + name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }} path: _packages/**/*.zip mac: @@ -63,32 +51,25 @@ jobs: strategy: fail-fast: false matrix: + profile: + - emqx + - emqx-ee + otp: + - 24.1.5-2 macos: - macos-11 - macos-10.15 - otp: - - 24.1.1-emqx-1 runs-on: ${{ matrix.macos }} steps: - - uses: actions/checkout@v1 - - name: prepare - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "${{ secrets.CI_GIT_TOKEN }}" >> ./scripts/git-token - echo "EMQX_NAME=emqx-ee" >> $GITHUB_ENV - else - echo "EMQX_NAME=emqx" >> $GITHUB_ENV - fi + - uses: actions/checkout@v2 - name: prepare run: | brew update brew install curl zip unzip gnu-sed kerl unixodbc freetds echo "/usr/local/bin" >> $GITHUB_PATH - git config --global credential.helper store + echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV - uses: actions/cache@v2 id: cache with: @@ -97,23 +78,23 @@ jobs: - name: build erlang if: steps.cache.outputs.cache-hit != 'true' timeout-minutes: 60 + env: + KERL_BUILD_BACKEND: git + OTP_GITHUB_URL: https://github.com/emqx/otp run: | - kerl build git https://github.com/emqx/otp.git OTP-${{ matrix.otp }} ${{ matrix.otp }} + kerl update releases + kerl build ${{ matrix.otp }} kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }} - - name: build + - name: build ${{ matrix.profile }} run: | . $HOME/.kerl/${{ matrix.otp }}/activate make ensure-rebar3 sudo cp rebar3 /usr/local/bin/rebar3 - make ${EMQX_NAME}-zip - - uses: actions/upload-artifact@v1 - if: failure() - with: - name: rebar3.crashdump - path: ./rebar3.crashdump + make ${{ matrix.profile }}-zip - name: test run: | - unzip -q $(find _packages/${EMQX_NAME} -mindepth 1 -maxdepth 1 -iname \*.zip | head) + pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip) + unzip -q $pkg_name # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins ./emqx/bin/emqx start || cat emqx/log/erlang.log.1 ready='no' diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index afcb384cb..c1a457665 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-ubuntu20.04" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index f08060ec2..fae85baa2 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -9,19 +9,9 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 1000 - - name: Set git token - if: endsWith(github.repository, 'enterprise') - run: | - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - name: Run elvis check run: | - set -e - if [ -f EMQX_ENTERPRISE ]; then - ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx-enterprise - else - ./scripts/elvis-check.sh $GITHUB_BASE_REF emqx - fi + ./scripts/elvis-check.sh $GITHUB_BASE_REF - name: Check line-break at EOF run: | ./scripts/check-nl-at-eof.sh diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index bbcec98c8..caa4e14ad 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -12,11 +12,11 @@ jobs: strategy: matrix: otp: - - "23.2.7.2-emqx-2" - - "24.1.1-emqx-1" + - 23.3.4.9-3 + - 24.1.5-2 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp }}-ubuntu20.04" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 46ce95dab..f64e4e21f 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -8,58 +8,27 @@ on: push: tags: - v* - - e* pull_request: jobs: prepare: - strategy: - matrix: - otp: - - "23.2.7.2-emqx-2" - - "24.1.1-emqx-1" - runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" - - outputs: - profile: ${{ steps.profile.outputs.profile }} + # prepare source with any OTP version, no need for a matrix + container: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 steps: - - name: get otp version - id: get_otp_version - run: | - otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)" - echo "::set-output name=otp::$otp" - uses: actions/checkout@v2 with: path: source fetch-depth: 0 - - name: set profile - id: profile - shell: bash - working-directory: source - run: | - vsn="$(./pkg-vsn.sh)" - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "::set-output name=profile::emqx-ee" - else - echo "::set-output name=profile::emqx" - fi - name: get deps - working-directory: source run: | - make ensure-rebar3 - ./rebar3 as default get-deps - rm -rf rebar.lock - - name: gen zip file - run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]* + make -C source deps-all + zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v2 with: - name: source-${{ steps.get_otp_version.outputs.otp }} - path: source-${{ steps.get_otp_version.outputs.otp }}.zip + name: source + path: source.zip docker_test: runs-on: ubuntu-20.04 @@ -68,35 +37,34 @@ jobs: strategy: fail-fast: false matrix: - otp: - - 23.2.7.2-emqx-2 - - 24.1.1-emqx-1 + profile: + - emqx-edge + - emqx + - emqx-ee cluster_db_backend: - - "mnesia" - - "rlog" + - mnesia + - rlog steps: - uses: actions/download-artifact@v2 with: - name: source-${{ matrix.otp }} + name: source path: . - name: unzip source code - run: unzip -q source-${{ matrix.otp }}.zip + run: unzip -q source.zip - name: make docker image working-directory: source env: - OTP: ${{ matrix.otp }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 run: | - make ${{ needs.prepare.outputs.profile }}-docker - echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV - echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV + make ${{ matrix.profile }}-docker - name: run emqx timeout-minutes: 5 working-directory: source run: | set -x - export EMQX_TEST_DB_BACKEND="${{ matrix.cluster_db_backend }}" - ./.ci/docker-compose-file/scripts/run-emqx.sh + IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh) + ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }} - name: make paho tests run: | if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then @@ -118,24 +86,24 @@ jobs: strategy: fail-fast: false matrix: - otp: - - 23.2.7.2-emqx-2 - - 24.1.1-emqx-1 + profile: + - emqx + # - emqx-ee # TODO test enterprise steps: - uses: actions/download-artifact@v2 with: - name: source-${{ matrix.otp }} + name: source path: . - name: unzip source code - run: unzip -q source-${{ matrix.otp }}.zip + run: unzip -q source.zip - name: make docker image working-directory: source env: - OTP: ${{ matrix.otp }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 run: | - make ${{ needs.prepare.outputs.profile }}-docker - echo "TARGET=emqx/${{ needs.prepare.outputs.profile }}" >> $GITHUB_ENV + make ${{ matrix.profile }}-docker + echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV - run: minikube start - name: run emqx on chart diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index f56121d8a..6a73f846b 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -15,12 +15,14 @@ jobs: relup_test: strategy: matrix: - otp: - - "23.2.7.2-emqx-2" - - "24.1.1-emqx-1" + profile: + - emqx + - emqx-ee + otp_vsn: + - 24.1.5-2 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-2:${{ matrix.otp_vsn }}-ubuntu20.04" defaults: run: @@ -43,7 +45,7 @@ jobs: - uses: actions/checkout@v2 with: repository: emqx/emqtt-bench - ref: 0.3.4 + ref: 0.3.4 path: emqtt-bench - uses: actions/checkout@v2 with: @@ -55,26 +57,18 @@ jobs: repository: ${{ github.repository }} path: emqx fetch-depth: 0 - - name: prepare - run: | - if make -C emqx emqx-ee --dry-run > /dev/null 2>&1; then - echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials - git config --global credential.helper store - echo "${{ secrets.CI_GIT_TOKEN }}" >> emqx/scripts/git-token - echo "PROFILE=emqx-ee" >> $GITHUB_ENV - else - echo "PROFILE=emqx" >> $GITHUB_ENV - fi - name: get version run: | set -e -x -u cd emqx + export PROFILE=${{ matrix.profile }} + export OTP_VSN=${{ matrix.otp_vsn }} + echo "PROFILE=$PROFILE" >> $GITHUB_ENV + echo "OTP_VSN=$OTP_VSN" >> $GITHUB_ENV if [ $PROFILE = "emqx" ];then broker="emqx-ce" - edition='opensource' else broker="emqx-ee" - edition='enterprise' fi echo "BROKER=$broker" >> $GITHUB_ENV @@ -82,11 +76,7 @@ jobs: echo "VSN=$vsn" >> $GITHUB_ENV pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')" - if [ $PROFILE = "emqx" ]; then - old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" - else - old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")" - fi + old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")" echo "OLD_VSNS=$old_vsns" >> $GITHUB_ENV - name: download emqx run: | @@ -95,7 +85,7 @@ jobs: cd emqx/_upgrade_base old_vsns=($(echo $OLD_VSNS | tr ' ' ' ')) for old_vsn in ${old_vsns[@]}; do - wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-ubuntu20.04-${old_vsn#[e|v]}-amd64.zip + wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.zip done - name: build emqx run: make -C emqx ${PROFILE}-zip diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index 5c52ed61e..f90250df3 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -15,12 +15,11 @@ jobs: run_static_analysis: strategy: matrix: - otp: - - "23.2.7.2-emqx-2" - - "24.1.1-emqx-1" + emqx_builder: + - 5.0-2:24.1.5-2 # run dialyzer on latest OTP runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04" steps: - uses: actions/checkout@v2 @@ -38,12 +37,11 @@ jobs: run_proper_test: strategy: matrix: - otp: - - "23.2.7.2-emqx-2" - - "24.1.1-emqx-1" + emqx_builder: + - 5.0-2:24.1.5-2 runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0:${{ matrix.otp }}-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.emqx_builder }}-ubuntu20.04" steps: - uses: actions/checkout@v2 @@ -67,32 +65,13 @@ jobs: steps: - uses: actions/checkout@v2 - - name: set edition - id: set_edition - run: | - if make emqx-ee --dry-run > /dev/null 2>&1; then - echo "EDITION=enterprise" >> $GITHUB_ENV - else - echo "EDITION=opensource" >> $GITHUB_ENV - fi - name: docker compose up - if: env.EDITION == 'opensource' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | docker-compose \ -f .ci/docker-compose-file/docker-compose.yaml \ up -d --build - - name: docker compose up - if: env.EDITION == 'enterprise' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - timeout-minutes: 20 - run: | - docker-compose \ - -f .ci/docker-compose-file/docker-compose.yaml \ - -f .ci/docker-compose-file/docker-compose-enterprise.yaml \ - up -d --build - name: run eunit run: | docker exec -i ${{ matrix.otp_release }} bash -c "make eunit" diff --git a/Makefile b/Makefile index b699bbe82..bb7024e88 100644 --- a/Makefile +++ b/Makefile @@ -3,16 +3,20 @@ REBAR_VERSION = 3.16.1-emqx-1 REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3.14 +export EMQX_DEFAULT_RUNNER = alpine:3.14 +export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh) -export EMQX_DESC ?= EMQ X export EMQX_DASHBOARD_VERSION ?= v5.0.0-beta.18 +export DOCKERFILE := deploy/docker/Dockerfile +export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing ifeq ($(OS),Windows_NT) export REBAR_COLOR=none endif PROFILE ?= emqx -REL_PROFILES := emqx emqx-edge -PKG_PROFILES := emqx-pkg emqx-edge-pkg +REL_PROFILES := emqx emqx-edge emqx-ee +PKG_PROFILES := emqx-pkg emqx-edge-pkg emqx-ee-pkg PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default CT_NODE_NAME ?= 'test@127.0.0.1' @@ -85,7 +89,6 @@ coveralls: $(REBAR) @ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send .PHONY: $(REL_PROFILES) - $(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs @$(REBAR) as $(@) do compile,release @@ -98,8 +101,10 @@ $(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs clean: $(PROFILES:%=clean-%) $(PROFILES:%=clean-%): @if [ -d _build/$(@:clean-%=%) ]; then \ + rm rebar.lock \ rm -rf _build/$(@:clean-%=%)/rel; \ find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \ + find _build/$(@:clean-%=%) -type l -delete; \ fi .PHONY: clean-all @@ -109,6 +114,7 @@ clean-all: .PHONY: deps-all deps-all: $(REBAR) $(PROFILES:%=deps-%) + @make clean # ensure clean at the end ## deps- is used in CI scripts to download deps and the ## share downloads between CI steps and/or copied into containers @@ -116,6 +122,7 @@ deps-all: $(REBAR) $(PROFILES:%=deps-%) .PHONY: $(PROFILES:%=deps-%) $(PROFILES:%=deps-%): $(REBAR) get-dashboard @$(REBAR) as $(@:deps-%=%) get-deps + @rm -f rebar.lock .PHONY: xref xref: $(REBAR) @@ -174,5 +181,17 @@ endef ALL_ZIPS = $(REL_PROFILES) $(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target,$(zt)))) +## emqx-docker-testing +## emqx-ee-docker-testing +## is to directly copy a unzipped zip-package to a +## base image such as ubuntu20.04. Mostly for testing +.PHONY: $(REL_PROFILES:%=%-docker-testing) +define gen-docker-target-testing +$1-docker-testing: $(COMMON_DEPS) + @$(BUILD) $1 docker-testing +endef +ALL_ZIPS = $(REL_PROFILES) +$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-docker-target-testing,$(zt)))) + conf-segs: @scripts/merge-config.escript diff --git a/apps/emqx/include/emqx_release.hrl b/apps/emqx/include/emqx_release.hrl index 5b4bc7d00..1424eb8a5 100644 --- a/apps/emqx/include/emqx_release.hrl +++ b/apps/emqx/include/emqx_release.hrl @@ -14,9 +14,6 @@ %% limitations under the License. %%-------------------------------------------------------------------- --ifndef(EMQX_RELEASE_HRL). --define(EMQX_RELEASE_HRL, true). - %% NOTE: this is the release version which is not always the same %% as the emqx app version defined in emqx.app.src %% App (plugin) versions are bumped independently. @@ -27,13 +24,4 @@ %% NOTE: This version number should be manually bumped for each release --ifndef(EMQX_ENTERPRISE). - --define(EMQX_RELEASE, {opensource, "5.0-beta.1"}). - --else. - - --endif. - --endif. +-define(EMQX_RELEASE, "5.0-beta.2"). diff --git a/apps/emqx/src/emqx.erl b/apps/emqx/src/emqx.erl index 1df7f0149..b2c73c9bb 100644 --- a/apps/emqx/src/emqx.erl +++ b/apps/emqx/src/emqx.erl @@ -20,6 +20,7 @@ -include("logger.hrl"). -include("types.hrl"). +-elvis([{elvis_style, god_modules, disable}]). %% Start/Stop the application -export([ start/0 @@ -51,10 +52,6 @@ , run_fold_hook/3 ]). -%% Troubleshooting --export([ set_debug_secret/1 - ]). - %% Configs APIs -export([ get_config/1 , get_config/2 @@ -71,29 +68,6 @@ -define(APP, ?MODULE). -%% @hidden Path to the file which has debug_info encryption secret in it. -%% Evaluate this function if there is a need to access encrypted debug_info. -%% NOTE: Do not change the API to accept the secret text because it may -%% get logged everywhere. -set_debug_secret(PathToSecretFile) -> - SecretText = - case file:read_file(PathToSecretFile) of - {ok, Secret} -> - try string:trim(binary_to_list(Secret)) - catch _ : _ -> error({badfile, PathToSecretFile}) - end; - {error, Reason} -> - ?ULOG("Failed to read debug_info encryption key file ~ts: ~p~n", - [PathToSecretFile, Reason]), - error(Reason) - end, - F = fun(init) -> ok; - (clear) -> ok; - ({debug_info, _Mode, _Module, _Filename}) -> SecretText - end, - _ = beam_lib:clear_crypto_key_fun(), - ok = beam_lib:crypto_key_fun(F). - %%-------------------------------------------------------------------- %% Bootstrap, is_running... %%-------------------------------------------------------------------- diff --git a/apps/emqx/src/emqx_app.erl b/apps/emqx/src/emqx_app.erl index d1090803d..4e130ef36 100644 --- a/apps/emqx/src/emqx_app.erl +++ b/apps/emqx/src/emqx_app.erl @@ -30,7 +30,6 @@ ]). -include("emqx.hrl"). --include("emqx_release.hrl"). -include("logger.hrl"). -define(APP, emqx). @@ -40,6 +39,7 @@ %%-------------------------------------------------------------------- start(_Type, _Args) -> + ok = emqx_release:put_edition(), ok = maybe_load_config(), ok = emqx_persistent_session:init_db_backend(), ok = maybe_start_quicer(), @@ -107,30 +107,7 @@ is_quicer_app_present() -> is_quic_listener_configured() -> emqx_listeners:has_enabled_listener_conf_by_type(quic). -get_description() -> - {ok, Descr0} = application:get_key(?APP, description), - case os:getenv("EMQX_DESCRIPTION") of - false -> Descr0; - "" -> Descr0; - Str -> string:strip(Str, both, $\n) - end. +get_description() -> emqx_release:description(). get_release() -> - case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of - false -> %% For TEST build or depedency build. - release_in_macro(); - {_, Vsn} -> %% For emqx release build - VsnStr = release_in_macro(), - case string:str(Vsn, VsnStr) of - 1 -> ok; - _ -> - erlang:error(#{ reason => version_mismatch - , source => VsnStr - , built_for => Vsn - }) - end, - Vsn - end. - -release_in_macro() -> - element(2, ?EMQX_RELEASE). + emqx_release:version(). diff --git a/apps/emqx/src/emqx_misc.erl b/apps/emqx/src/emqx_misc.erl index 446039778..0d3edd551 100644 --- a/apps/emqx/src/emqx_misc.erl +++ b/apps/emqx/src/emqx_misc.erl @@ -17,6 +17,7 @@ -module(emqx_misc). -compile(inline). +-elvis([{elvis_style, god_modules, disable}]). -include("types.hrl"). -include("logger.hrl"). @@ -65,21 +66,13 @@ maybe_parse_ip(Host) -> end. %% @doc Add `ipv6_probe' socket option if it's supported. +%% gen_tcp:ipv6_probe() -> true. is added to EMQ's OTP forks ipv6_probe(Opts) -> - case persistent_term:get({?MODULE, ipv6_probe_supported}, unknown) of - unknown -> - %% e.g. 23.2.7.1-emqx-2-x86_64-unknown-linux-gnu-64 - OtpVsn = emqx_vm:get_otp_version(), - Bool = (match =:= re:run(OtpVsn, "emqx", [{capture, none}])), - _ = persistent_term:put({?MODULE, ipv6_probe_supported}, Bool), - ipv6_probe(Bool, Opts); - Bool -> - ipv6_probe(Bool, Opts) + case erlang:function_exported(gen_tcp, ipv6_probe, 0) of + true -> [{ipv6_probe, true} | Opts]; + false -> Opts end. -ipv6_probe(false, Opts) -> Opts; -ipv6_probe(true, Opts) -> [{ipv6_probe, true} | Opts]. - %% @doc Merge options -spec(merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist()). merge_opts(Defaults, Options) -> @@ -100,9 +93,9 @@ maybe_apply(Fun, Arg) when is_function(Fun) -> -spec(compose(list(F)) -> G when F :: fun((any()) -> any()), G :: fun((any()) -> any())). -compose([F|More]) -> compose(F, More). +compose([F | More]) -> compose(F, More). --spec(compose(F, G|[Gs]) -> C +-spec(compose(F, G | [Gs]) -> C when F :: fun((X1) -> X2), G :: fun((X2) -> X3), Gs :: [fun((Xn) -> Xn1)], @@ -110,19 +103,19 @@ compose([F|More]) -> compose(F, More). X3 :: any(), Xn :: any(), Xn1 :: any(), Xm :: any()). compose(F, G) when is_function(G) -> fun(X) -> G(F(X)) end; compose(F, [G]) -> compose(F, G); -compose(F, [G|More]) -> compose(compose(F, G), More). +compose(F, [G | More]) -> compose(compose(F, G), More). %% @doc RunFold run_fold([], Acc, _State) -> Acc; -run_fold([Fun|More], Acc, State) -> +run_fold([Fun | More], Acc, State) -> run_fold(More, Fun(Acc, State), State). %% @doc Pipeline pipeline([], Input, State) -> {ok, Input, State}; -pipeline([Fun|More], Input, State) -> +pipeline([Fun | More], Input, State) -> case apply_fun(Fun, Input, State) of ok -> pipeline(More, Input, State); {ok, NState} -> @@ -171,7 +164,7 @@ drain_deliver(0, Acc) -> drain_deliver(N, Acc) -> receive Deliver = {deliver, _Topic, _Msg} -> - drain_deliver(N-1, [Deliver|Acc]) + drain_deliver(N-1, [Deliver | Acc]) after 0 -> lists:reverse(Acc) end. @@ -186,7 +179,7 @@ drain_down(0, Acc) -> drain_down(Cnt, Acc) -> receive {'DOWN', _MRef, process, Pid, _Reason} -> - drain_down(Cnt-1, [Pid|Acc]) + drain_down(Cnt-1, [Pid | Acc]) after 0 -> lists:reverse(Acc) end. @@ -213,7 +206,7 @@ check_oom(Pid, #{max_message_queue_len := MaxQLen, end. do_check_oom([]) -> ok; -do_check_oom([{Val, Max, Reason}|Rest]) -> +do_check_oom([{Val, Max, Reason} | Rest]) -> case is_integer(Max) andalso (0 < Max) andalso (Max < Val) of true -> {shutdown, Reason}; false -> do_check_oom(Rest) @@ -256,8 +249,8 @@ proc_stats(Pid) -> reductions, memory]) of undefined -> []; - [{message_queue_len, Len}|ProcStats] -> - [{mailbox_len, Len}|ProcStats] + [{message_queue_len, Len} | ProcStats] -> + [{mailbox_len, Len} | ProcStats] end. rand_seed() -> @@ -277,9 +270,9 @@ index_of(E, L) -> index_of(_E, _I, []) -> error(badarg); -index_of(E, I, [E|_]) -> +index_of(E, I, [E | _]) -> I; -index_of(E, I, [_|L]) -> +index_of(E, I, [_ | L]) -> index_of(E, I+1, L). -spec(bin2hexstr_A_F(binary()) -> binary()). @@ -339,6 +332,12 @@ pad(L, Count) -> -include_lib("eunit/include/eunit.hrl"). ipv6_probe_test() -> - ?assertEqual([{ipv6_probe, true}], ipv6_probe([])). + try gen_tcp:ipv6_probe() of + true -> + ?assertEqual([{ipv6_probe, true}], ipv6_probe([])) + catch + _ : _ -> + ok + end. -endif. diff --git a/apps/emqx/src/emqx_release.erl b/apps/emqx/src/emqx_release.erl new file mode 100644 index 000000000..1e362d0f0 --- /dev/null +++ b/apps/emqx/src/emqx_release.erl @@ -0,0 +1,86 @@ +%%-------------------------------------------------------------------- +%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%-------------------------------------------------------------------- + +-module(emqx_release). + +-export([ edition/0 + , put_edition/0 + , put_edition/1 + , description/0 + , version/0 + ]). + +-include("emqx_release.hrl"). + +%% @doc Return EMQ X description. +description() -> + case os:getenv("EMQX_DESCRIPTION") of + false -> "EMQ X Community Edition"; + "" -> "EMQ X Community Edition"; + Str -> string:strip(Str, both, $\n) + end. + +%% @doc Return EMQ X edition info. +%% Read info from persistent_term at runtime. +%% Or meck this function to run tests for another eidtion. +-spec edition() -> ce | ee | edge. +edition() -> + try persistent_term:get(emqx_edition) + catch error : badarg -> get_edition() end. + +%% @private initiate EMQ X edition info in persistent_term. +put_edition() -> + ok = put_edition(get_edition()). + +%% @hidden This function is mostly for testing. +%% Switch to another eidtion at runtime to run edition-specific tests. +-spec put_edition(ce | ee | edge) -> ok. +put_edition(Which) -> + persistent_term:put(emqx_edition, Which), + ok. + +-spec get_edition() -> ce | ee | edge. +get_edition() -> + edition(description()). + +edition(Desc) -> + case re:run(Desc, "enterprise", [caseless]) of + {match, _} -> + ee; + _ -> + case re:run(Desc, "edge", [caseless]) of + {match, _} -> edge; + _ -> ce + end + end. + +%% @doc Return the release version. +version() -> + case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of + false -> %% For TEST build or depedency build. + ?EMQX_RELEASE; + {_, Vsn} -> %% For emqx release build + VsnStr = ?EMQX_RELEASE, + case string:str(Vsn, VsnStr) of + 1 -> ok; + _ -> + erlang:error(#{ reason => version_mismatch + , source => VsnStr + , built_for => Vsn + }) + end, + Vsn + end. diff --git a/apps/emqx/test/emqx_authentication_SUITE.erl b/apps/emqx/test/emqx_authentication_SUITE.erl index e04b8566b..21a59e0c9 100644 --- a/apps/emqx/test/emqx_authentication_SUITE.erl +++ b/apps/emqx/test/emqx_authentication_SUITE.erl @@ -270,7 +270,7 @@ t_restart(Config) when is_list(Config) -> ?assertEqual({ok, [test_chain]}, ?AUTHN:list_chain_names()); -t_restart({'end', Config}) -> +t_restart({'end', _Config}) -> ?AUTHN:delete_chain(test_chain), ok. diff --git a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl index 3b32eccfd..8f40427e5 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_admin.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_admin.erl @@ -171,7 +171,7 @@ check(_, undefined) -> check(Username, Password) -> case lookup_user(Username) of [#?ADMIN{pwdhash = <>}] -> - case Hash =:= sha3_hash(Salt, Password) of + case Hash =:= sha256(Salt, Password) of true -> ok; false -> {error, <<"BAD_USERNAME_OR_PASSWORD">>} end; @@ -206,10 +206,10 @@ destroy_token_by_username(Username, Token) -> hash(Password) -> SaltBin = emqx_dashboard_token:salt(), - <>. + <>. -sha3_hash(SaltBin, Password) -> - crypto:hash('sha3_256', <>). +sha256(SaltBin, Password) -> + crypto:hash('sha256', <>). add_default_user() -> add_default_user(binenv(default_username), binenv(default_password)). diff --git a/apps/emqx_dashboard/src/emqx_dashboard_api.erl b/apps/emqx_dashboard/src/emqx_dashboard_api.erl index cb3545da3..c40c4bd22 100644 --- a/apps/emqx_dashboard/src/emqx_dashboard_api.erl +++ b/apps/emqx_dashboard/src/emqx_dashboard_api.erl @@ -16,16 +16,6 @@ -module(emqx_dashboard_api). --ifndef(EMQX_ENTERPRISE). - --define(RELEASE, community). - --else. - --define(VERSION, enterprise). - --endif. - -behaviour(minirest_api). -include("emqx_dashboard.hrl"). @@ -200,7 +190,10 @@ login(post, #{body := Params}) -> case emqx_dashboard_admin:sign_token(Username, Password) of {ok, Token} -> Version = iolist_to_binary(proplists:get_value(version, emqx_sys:info())), - {200, #{token => Token, version => Version, license => #{edition => ?RELEASE}}}; + {200, #{token => Token, + version => Version, + license => #{edition => emqx_release:edition()} + }}; {error, _} -> {401, #{code => ?ERROR_USERNAME_OR_PWD, message => <<"Auth filed">>}} end. diff --git a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl index 146a4e8e5..061f7c839 100644 --- a/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl +++ b/apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl @@ -150,11 +150,11 @@ t_cli(_Config) -> emqx_dashboard_cli:admins(["add", "username", "password"]), [#?ADMIN{ username = <<"username">>, pwdhash = <>}] = emqx_dashboard_admin:lookup_user(<<"username">>), - ?assertEqual(Hash, crypto:hash(sha3_256, <>/binary>>)), + ?assertEqual(Hash, crypto:hash(sha256, <>/binary>>)), emqx_dashboard_cli:admins(["passwd", "username", "newpassword"]), [#?ADMIN{username = <<"username">>, pwdhash = <>}] = emqx_dashboard_admin:lookup_user(<<"username">>), - ?assertEqual(Hash1, crypto:hash(sha3_256, <>/binary>>)), + ?assertEqual(Hash1, crypto:hash(sha256, <>/binary>>)), emqx_dashboard_cli:admins(["del", "username"]), [] = emqx_dashboard_admin:lookup_user(<<"username">>), emqx_dashboard_cli:admins(["add", "admin1", "pass1"]), diff --git a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl index c94242a99..06b710492 100644 --- a/apps/emqx_rule_engine/src/emqx_rule_funcs.erl +++ b/apps/emqx_rule_engine/src/emqx_rule_funcs.erl @@ -884,29 +884,27 @@ time_unit(<<"nanosecond">>) -> nanosecond. %% Here the emqx_rule_funcs module acts as a proxy, forwarding %% the function handling to the worker module. %% @end --ifdef(EMQX_ENTERPRISE). -'$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> - emqx_schema_parser:decode(SchemaId, Data, MoreArgs); -'$handle_undefined_function'(schema_decode, Args) -> - error({args_count_error, {schema_decode, Args}}); +% '$handle_undefined_function'(schema_decode, [SchemaId, Data|MoreArgs]) -> +% emqx_schema_parser:decode(SchemaId, Data, MoreArgs); +% '$handle_undefined_function'(schema_decode, Args) -> +% error({args_count_error, {schema_decode, Args}}); -'$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> - emqx_schema_parser:encode(SchemaId, Term, MoreArgs); -'$handle_undefined_function'(schema_encode, Args) -> - error({args_count_error, {schema_encode, Args}}); +% '$handle_undefined_function'(schema_encode, [SchemaId, Term|MoreArgs]) -> +% emqx_schema_parser:encode(SchemaId, Term, MoreArgs); +% '$handle_undefined_function'(schema_encode, Args) -> +% error({args_count_error, {schema_encode, Args}}); + +% '$handle_undefined_function'(sprintf, [Format|Args]) -> +% erlang:apply(fun sprintf_s/2, [Format, Args]); + +% '$handle_undefined_function'(Fun, Args) -> +% error({sql_function_not_supported, function_literal(Fun, Args)}). '$handle_undefined_function'(sprintf, [Format|Args]) -> erlang:apply(fun sprintf_s/2, [Format, Args]); '$handle_undefined_function'(Fun, Args) -> error({sql_function_not_supported, function_literal(Fun, Args)}). --else. -'$handle_undefined_function'(sprintf, [Format|Args]) -> - erlang:apply(fun sprintf_s/2, [Format, Args]); - -'$handle_undefined_function'(Fun, Args) -> - error({sql_function_not_supported, function_literal(Fun, Args)}). --endif. % EMQX_ENTERPRISE map_path(Key) -> {path, [{key, P} || P <- string:split(Key, ".", all)]}. diff --git a/build b/build index 1e4fac8c8..7e23d3113 100755 --- a/build +++ b/build @@ -12,23 +12,10 @@ ARTIFACT="$2" # ensure dir cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" -PKG_VSN="$(./pkg-vsn.sh)" +PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}" export PKG_VSN -if [ "$(uname -s)" = 'Darwin' ]; then - DIST='macos' - VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') - SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" -elif [ "$(uname -s)" = 'Linux' ]; then - if grep -q -i 'centos' /etc/*-release; then - DIST='centos' - VERSION_ID="$(rpm --eval '%{centos_ver}')" - else - DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" - VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" - fi - SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" -fi +SYSTEM="$(./scripts/get-distro.sh)" ARCH="$(uname -m)" case "$ARCH" in @@ -48,8 +35,8 @@ export ARCH ## Support RPM and Debian based linux systems ## if [ "$(uname -s)" = 'Linux' ]; then - case "${DIST:-}" in - ubuntu|debian|raspbian) + case "${SYSTEM:-}" in + ubuntu*|debian*|raspbian*) PKGERDIR='deb' ;; *) @@ -109,7 +96,7 @@ make_relup() { rm -rf "$tmp_dir" fi releases+=( "$base_vsn" ) - done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-$SYSTEM*-$ARCH.zip" -type f) + done < <(find _upgrade_base -maxdepth 1 -name "*$PROFILE-otp${OTP_VSN}-$SYSTEM*-$ARCH.zip" -type f) fi if [ ${#releases[@]} -eq 0 ]; then log "No upgrade base found, relup ignored" @@ -149,7 +136,7 @@ make_zip() { log "ERROR: $tarball is not found" fi local zipball - zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-${ARCH}.zip" + zipball="${pkgpath}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" tar zxf "${tarball}" -C "${tard}/emqx" ## try to be portable for zip packages. ## for DEB and RPM packages the dependencies are resoved by yum and apt @@ -157,18 +144,60 @@ make_zip() { (cd "${tard}" && zip -qr - emqx) > "${zipball}" } +## This function builds the default docker image based on alpine:3.14 (by default) make_docker() { - ## Build Docker image - echo "DOCKER BUILD: Build Docker image." - echo "DOCKER BUILD: build version -> $PKG_VSN." - echo "DOCKER BUILD: docker repo -> emqx/$PROFILE " - - docker build --no-cache \ - --build-arg BUILD_FROM="ghcr.io/emqx/emqx-builder/5.0:${OTP:-24.1.1-emqx-1}-alpine3.14" \ - --build-arg RUN_FROM="alpine:3.14" \ + EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}" + EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}" + set -x + docker build --no-cache --pull \ + --build-arg BUILD_FROM="${EMQX_BUILDER}" \ + --build-arg RUN_FROM="${EMQX_RUNNER}" \ --build-arg EMQX_NAME="$PROFILE" \ - --tag "emqx/$PROFILE:$PKG_VSN" \ - -f deploy/docker/Dockerfile . + --tag "emqx/$PROFILE:${PKG_VSN}" \ + -f "${DOCKERFILE}" . +} + +## This function accepts any base docker image, +## a emqx zip-image, and a image tag (for the image to be built), +## to build a docker image which runs EMQ X +## +## Export below variables to quickly build an image +## +## Name Default Example +## --------------------------------------------------------------------- +## EMQX_BASE_IMAGE current os centos:7 +## EMQX_ZIP_PACKAGE _packages/ /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.zip +## EMQX_IMAGE_TAG emqx/emqx: emqx/emqx:testing-tag +## +make_docker_testing() { + if [ -z "${EMQX_BASE_IMAGE:-}" ]; then + case "$SYSTEM" in + ubuntu20*) + EMQX_BASE_IMAGE="ubuntu:20.04" + ;; + centos8) + EMQX_BASE_IMAGE="centos:8" + ;; + *) + echo "Unsupported testing base image for $SYSTEM" + exit 1 + ;; + esac + fi + EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}" + local defaultzip + defaultzip="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.zip" + local zip="${EMQX_ZIP_PACKAGE:-$defaultzip}" + if [ ! -f "$zip" ]; then + log "ERROR: $zip not built?" + exit 1 + fi + set -x + docker build \ + --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \ + --build-arg EMQX_ZIP_PACKAGE="${zip}" \ + --tag "$EMQX_IMAGE_TAG" \ + -f "${DOCKERFILE_TESTING}" . } log "building artifact=$ARTIFACT for profile=$PROFILE" @@ -194,6 +223,9 @@ case "$ARTIFACT" in docker) make_docker ;; + docker-testing) + make_docker_testing + ;; *) log "Unknown artifact $ARTIFACT" exit 1 diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index b92cffa9c..1dd7d318a 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0:24.1.1-emqx-1-alpine3.14 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-2:24.1.5-2-alpine3.14 ARG RUN_FROM=alpine:3.14 FROM ${BUILD_FROM} AS builder diff --git a/deploy/packages/deb/Makefile b/deploy/packages/deb/Makefile index 48124d780..1f709d860 100644 --- a/deploy/packages/deb/Makefile +++ b/deploy/packages/deb/Makefile @@ -8,7 +8,7 @@ EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz SOURCE_PKG := $(EMQX_NAME)_$(PKG_VSN)_$(shell dpkg --print-architecture) -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) .PHONY: all all: | $(BUILT) diff --git a/deploy/packages/deb/debian/control b/deploy/packages/deb/debian/control index e35535c12..b3794036c 100644 --- a/deploy/packages/deb/debian/control +++ b/deploy/packages/deb/debian/control @@ -4,7 +4,7 @@ Priority: optional Maintainer: emqx Build-Depends: debhelper (>=9) Standards-Version: 3.9.6 -Homepage: https://www.emqx.io +Homepage: https://www.emqx.com Package: emqx Architecture: any diff --git a/deploy/packages/rpm/Makefile b/deploy/packages/rpm/Makefile index 5a6e6bee4..acee8b51c 100644 --- a/deploy/packages/rpm/Makefile +++ b/deploy/packages/rpm/Makefile @@ -5,8 +5,9 @@ BUILT := $(SRCDIR)/BUILT dash := - none := space := $(none) $(none) -RPM_VSN ?= $(shell echo $(PKG_VSN) | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?") -RPM_REL ?= $(shell echo $(PKG_VSN) | grep -oE "(alpha|beta|rc)\.[0-9]") +## RPM does not allow '-' in version nubmer and release string, replace with '_' +RPM_VSN := $(subst -,_,$(PKG_VSN)) +RPM_REL := otp$(subst -,_,$(OTP_VSN)) ARCH ?= amd64 ifeq ($(ARCH),mips64) @@ -16,12 +17,8 @@ endif EMQX_NAME=$(subst -pkg,,$(EMQX_BUILD)) TAR_PKG := $(EMQX_REL)/_build/$(EMQX_BUILD)/rel/emqx/emqx-$(PKG_VSN).tar.gz -TARGET_PKG := $(EMQX_NAME)-$(SYSTEM)-$(PKG_VSN)-$(ARCH) -ifeq ($(RPM_REL),) - # no tail - RPM_REL := 1 -endif -SOURCE_PKG := emqx-$(SYSTEM)-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) +TARGET_PKG := $(EMQX_NAME)-$(PKG_VSN)-otp$(OTP_VSN)-$(SYSTEM)-$(ARCH) +SOURCE_PKG := emqx-$(RPM_VSN)-$(RPM_REL).$(shell uname -m) SYSTEMD := $(shell if command -v systemctl >/dev/null 2>&1; then echo yes; fi) # Not $(PWD) as it does not work for make -C @@ -47,7 +44,6 @@ all: | $(BUILT) --define "_service_dst $(SERVICE_DST)" \ --define "_post_addition $(POST_ADDITION)" \ --define "_preun_addition $(PREUN_ADDITION)" \ - --define "_ostype -$(SYSTEM)" \ --define "_sharedstatedir /var/lib" \ emqx.spec mkdir -p $(EMQX_REL)/_packages/$(EMQX_NAME) diff --git a/deploy/packages/rpm/emqx.spec b/deploy/packages/rpm/emqx.spec index 44e02ea45..c6eb56a6f 100644 --- a/deploy/packages/rpm/emqx.spec +++ b/deploy/packages/rpm/emqx.spec @@ -5,7 +5,7 @@ %define _log_dir %{_var}/log/%{_name} %define _lib_home /usr/lib/%{_name} %define _var_home %{_sharedstatedir}/%{_name} -%define _build_name_fmt %{_arch}/%{_name}%{?_ostype}-%{_version}-%{_release}.%{_arch}.rpm +%define _build_name_fmt %{_arch}/%{_name}-%{_version}-%{_release}.%{_arch}.rpm %define _build_id_links none Name: %{_package_name} diff --git a/lib-ee/.gitkeep b/lib-ee/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/pkg-vsn.sh b/pkg-vsn.sh index 2b8ac6fd1..ad24bb974 100755 --- a/pkg-vsn.sh +++ b/pkg-vsn.sh @@ -6,14 +6,8 @@ set -euo pipefail # ensure dir cd -P -- "$(dirname -- "$0")" -if [ -f EMQX_ENTERPRISE ]; then - EDITION='enterprise' -else - EDITION='opensource' -fi - ## emqx_release.hrl is the single source of truth for release version -RELEASE="$(grep -E "define.+EMQX_RELEASE.+${EDITION}" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)" +RELEASE="$(grep -E "define.+EMQX_RELEASE" apps/emqx/include/emqx_release.hrl | cut -d '"' -f2)" git_exact_vsn() { local tag diff --git a/rebar.config.erl b/rebar.config.erl index 1b82e281f..78e4f3e41 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -130,18 +130,17 @@ test_deps() -> , {proper, "1.4.0"} ]. -common_compile_opts(Edition) -> +common_compile_opts() -> [ debug_info % alwyas include debug_info , {compile_info, [{emqx_vsn, get_vsn()}]} ] ++ - [{d, 'EMQX_ENTERPRISE'} || is_enterprise(Edition)] ++ [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ]. -prod_compile_opts(Edition) -> +prod_compile_opts() -> [ compressed , deterministic , warnings_as_errors - | common_compile_opts(Edition) + | common_compile_opts() ]. prod_overrides() -> @@ -149,34 +148,41 @@ prod_overrides() -> profiles() -> Vsn = get_vsn(), - ce_profiles(Vsn) ++ ee_profiles(Vsn). - -ce_profiles(Vsn) -> - [ {'emqx', [ {erl_opts, prod_compile_opts(ce)} + [ {'emqx', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} ]} - , {'emqx-pkg', [ {erl_opts, prod_compile_opts(ce)} + , {'emqx-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, cloud, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} ]} - , {'emqx-edge', [ {erl_opts, prod_compile_opts(ce)} + , {'emqx-ee', [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, bin, ee)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ee)} + ]} + , {'emqx-ee-pkg', [ {erl_opts, prod_compile_opts()} + , {relx, relx(Vsn, cloud, pkg, ee)} + , {overrides, prod_overrides()} + , {project_app_dirs, project_app_dirs(ee)} + ]} + , {'emqx-edge', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, bin, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} ]} - , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts(ce)} + , {'emqx-edge-pkg', [ {erl_opts, prod_compile_opts()} , {relx, relx(Vsn, edge, pkg, ce)} , {overrides, prod_overrides()} , {project_app_dirs, project_app_dirs(ce)} ]} - , {check, [ {erl_opts, common_compile_opts(ce)} + , {check, [ {erl_opts, common_compile_opts()} , {project_app_dirs, project_app_dirs(ce)} ]} , {test, [ {deps, test_deps()} - , {erl_opts, common_compile_opts(ce) ++ erl_opts_i(ce) } + , {erl_opts, common_compile_opts() ++ erl_opts_i(ce) } , {extra_src_dirs, [{"test", [{recursive, true}]}]} , {project_app_dirs, project_app_dirs(ce)} ]} @@ -198,13 +204,11 @@ relx(Vsn, RelType, PkgType, Edition) -> | overlay_vars(RelType, PkgType, Edition)]} ]. -emqx_description(cloud, ee) -> "EMQ X Enterprise"; -emqx_description(cloud, ce) -> "EMQ X Broker"; -emqx_description(edge, ce) -> "EMQ X Edge". +emqx_description(cloud, ee) -> "EMQ X Enterprise Edition"; +emqx_description(cloud, ce) -> "EMQ X Community Edition"; +emqx_description(edge, ce) -> "EMQ X Edge Edition". -overlay_vars(_RelType, PkgType, ee) -> - ee_overlay_vars(PkgType); -overlay_vars(RelType, PkgType, ce) -> +overlay_vars(RelType, PkgType, _Edition) -> overlay_vars_rel(RelType) ++ overlay_vars_pkg(PkgType). %% vars per release type, cloud or edge @@ -289,7 +293,7 @@ relx_apps(ReleaseType, Edition) -> , emqx_limiter ] ++ [quicer || is_quicer_supported()] - ++ [emqx_license || is_enterprise(Edition)] + %++ [emqx_license || is_enterprise(Edition)] ++ [bcrypt || provide_bcrypt_release(ReleaseType)] ++ relx_apps_per_rel(ReleaseType) %% NOTE: applications below are only loaded after node start/restart @@ -472,38 +476,3 @@ list_dir(Dir) -> false -> [] end. - -%% ==== Enterprise supports below ================================================================== - -ee_profiles(Vsn) -> - [ {'emqx-ee', [ {erl_opts, prod_compile_opts(ee)} - , {relx, relx(Vsn, cloud, bin, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {'emqx-ee-pkg', [ {erl_opts, prod_compile_opts(ee)} - , {relx, relx(Vsn, cloud, pkg, ee)} - , {overrides, prod_overrides()} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {'check-ee', [ {erl_opts, common_compile_opts(ee)} - , {project_app_dirs, project_app_dirs(ee)} - ]} - , {'test-ee', [ {deps, test_deps()} - , {erl_opts, common_compile_opts(ee) ++ erl_opts_i(ee) } - , {extra_src_dirs, [{"test", [{recursive, true}]}]} - , {project_app_dirs, project_app_dirs(ee)} - ]} - ]. - -ee_overlay_vars(PkgType) -> - Common = [], - Common ++ ee_overlay_vars_pkg(PkgType). - -%% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb) -ee_overlay_vars_pkg(bin) -> - [ - ]; -ee_overlay_vars_pkg(pkg) -> - [ - ]. diff --git a/scripts/check-deps-integrity.escript b/scripts/check-deps-integrity.escript index 3cc8fdc53..43223701b 100755 --- a/scripts/check-deps-integrity.escript +++ b/scripts/check-deps-integrity.escript @@ -7,12 +7,7 @@ main([]) -> Files = ["rebar.config"] ++ apps_rebar_config("apps") ++ - case filelib:is_file("EMQX_ENTERPRISE") of - true -> - true = filelib:is_dir("lib-ee"), - apps_rebar_config("lib-ee"); - false -> [] - end, + apps_rebar_config("lib-ee"), Deps = collect_deps(Files, #{}), case count_bad_deps(Deps) of 0 -> diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index 47b71c7ca..003ffbc5e 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -11,9 +11,7 @@ find_app() { } find_app 'apps' -if [ -f 'EMQX_ENTERPRISE' ]; then - find_app 'lib-ee' -fi +find_app 'lib-ee' ## find directories in lib-extra find_app 'lib-extra' diff --git a/scripts/get-distro.sh b/scripts/get-distro.sh new file mode 100755 index 000000000..00e95e1d8 --- /dev/null +++ b/scripts/get-distro.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +## This script prints Linux distro name and its version number +## e.g. macos, centos8, ubuntu20.04 + +set -euo pipefail + +if [ "$(uname -s)" = 'Darwin' ]; then + DIST='macos' + VERSION_ID=$(sw_vers | gsed -n '/^ProductVersion:/p' | gsed -r 's/ProductVersion:(.*)/\1/g' | gsed -r 's/([0-9]+).*/\1/g' | gsed 's/^[ \t]*//g') + SYSTEM="$(echo "${DIST}${VERSION_ID}" | gsed -r 's/([a-zA-Z]*)-.*/\1/g')" +elif [ "$(uname -s)" = 'Linux' ]; then + if grep -q -i 'centos' /etc/*-release; then + DIST='centos' + VERSION_ID="$(rpm --eval '%{centos_ver}')" + else + DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" + VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')" + fi + SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')" +fi +echo "$SYSTEM" diff --git a/scripts/get-otp-vsn.sh b/scripts/get-otp-vsn.sh new file mode 100755 index 000000000..a791318dc --- /dev/null +++ b/scripts/get-otp-vsn.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +erl -noshell -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' diff --git a/scripts/git-hook-pre-push.sh b/scripts/git-hook-pre-push.sh index 5b3f2edf2..60df47333 100755 --- a/scripts/git-hook-pre-push.sh +++ b/scripts/git-hook-pre-push.sh @@ -4,6 +4,8 @@ set -euo pipefail url="$2" +# we keep this to secure OLD private repo +# even after we have disclosed new code under EMQ BSL 1.0 if [ -f 'EMQX_ENTERPRISE' ]; then if [[ "$url" != *emqx-enterprise* ]]; then echo "$(tput setaf 1)error: enterprise_code_to_non_enterprise_repo"