diff --git a/.ci/docker-compose-file/Makefile.local b/.ci/docker-compose-file/Makefile.local index d11ab64a6..ff4f348b0 100644 --- a/.ci/docker-compose-file/Makefile.local +++ b/.ci/docker-compose-file/Makefile.local @@ -2,7 +2,7 @@ define usage make -f .ci/docker-compose-file/Makefile.local up -make -f .ci/docker-compose-file/Makefile.local ct CONTAINER=erlang24 SUITE=apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl +make -f .ci/docker-compose-file/Makefile.local ct CONTAINER=erlang SUITE=apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl make -f .ci/docker-compose-file/Makefile.local down endef export usage diff --git a/.ci/docker-compose-file/docker-compose-kafka.yaml b/.ci/docker-compose-file/docker-compose-kafka.yaml index ba0161293..716c4c709 100644 --- a/.ci/docker-compose-file/docker-compose-kafka.yaml +++ b/.ci/docker-compose-file/docker-compose-kafka.yaml @@ -19,7 +19,7 @@ services: command: /bin/generate-certs.sh kdc: hostname: kdc.emqx.net - image: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + image: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04 container_name: kdc.emqx.net networks: emqx_bridge: diff --git a/.ci/docker-compose-file/docker-compose.yaml b/.ci/docker-compose-file/docker-compose.yaml index 4a5ef7070..b55b3196f 100644 --- a/.ci/docker-compose-file/docker-compose.yaml +++ b/.ci/docker-compose-file/docker-compose.yaml @@ -1,9 +1,9 @@ version: '3.9' services: - erlang24: - container_name: erlang24 - image: ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04 + erlang: + container_name: erlang + image: ${DOCKER_CT_RUNNER_IMAGE:-ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04} env_file: - conf.env environment: diff --git a/.github/actions/package-macos/action.yaml b/.github/actions/package-macos/action.yaml index 37284e8b6..acd4975e6 100644 --- a/.github/actions/package-macos/action.yaml +++ b/.github/actions/package-macos/action.yaml @@ -3,7 +3,7 @@ inputs: profile: # emqx, emqx-enterprise required: true type: string - otp: # 24.2.1-1, 23.3.4.9-3 + otp: # 25.1.2-2, 24.3.4.2-1 required: true type: string os: diff --git a/.github/workflows/build_and_push_docker_images.yaml b/.github/workflows/build_and_push_docker_images.yaml index 2893489e4..76238c75f 100644 --- a/.github/workflows/build_and_push_docker_images.yaml +++ b/.github/workflows/build_and_push_docker_images.yaml @@ -23,7 +23,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} @@ -119,8 +119,10 @@ jobs: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] # NOTE: 'otp' and 'elixir' are to configure emqx-builder image # only support latest otp and elixir, not a matrix + builder: + - 5.0-26 # update to latest otp: - - 24.3.4.2-1 # update to latest + - 24.3.4.2-1 # switch to 25 once ready to release 5.1 elixir: - 1.13.4 # update to latest exclude: # TODO: publish enterprise to ecr too? @@ -174,7 +176,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} @@ -208,8 +210,10 @@ jobs: - 'docker.io' os: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] + builder: + - 5.0-26 # update to latest otp: - - 24.3.4.2-1 # update to latest + - 25.1.2-2 # update to latest elixir: - 1.13.4 # update to latest @@ -252,7 +256,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} RUN_FROM=${{ matrix.os[1] }} EMQX_NAME=${{ steps.meta.outputs.emqx_name }} file: source/${{ matrix.os[2] }} @@ -277,7 +281,7 @@ jobs: - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"] # NOTE: only support latest otp version, not a matrix otp: - - 24.3.4.2-1 # update to latest + - 24.3.4.2-1 # switch to 25 once ready to release 5.1 registry: - 'docker.io' - 'public.ecr.aws' @@ -342,7 +346,7 @@ jobs: - ${{ needs.prepare.outputs.BUILD_PROFILE }} # NOTE: for docker, only support latest otp version, not a matrix otp: - - 24.3.4.2-1 # update to latest + - 25.1.2-2 # update to latest elixir: - 1.13.4 # update to latest registry: diff --git a/.github/workflows/build_packages.yaml b/.github/workflows/build_packages.yaml index 80a8f1468..2bb4fb42e 100644 --- a/.github/workflows/build_packages.yaml +++ b/.github/workflows/build_packages.yaml @@ -23,7 +23,7 @@ on: jobs: prepare: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04 outputs: BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }} IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }} @@ -48,7 +48,7 @@ jobs: echo "WARN: This is NOT an exact git tag, will not publish release" is_exact_tag='false' fi - echo "::set-output name=IS_EXACT_TAG::${is_exact_tag}" + echo "IS_EXACT_TAG=${is_exact_tag}" >> $GITHUB_OUTPUT case $tag in refs/tags/v*) PROFILE='emqx' @@ -103,7 +103,7 @@ jobs: - uses: ilammy/msvc-dev-cmd@v1.12.0 - uses: erlef/setup-beam@v1 with: - otp-version: 24.2.1 + otp-version: 24.3.4 - name: build env: PYTHON: python @@ -173,13 +173,15 @@ jobs: needs: prepare runs-on: ${{ matrix.build_machine }} container: - image: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" + image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}" strategy: fail-fast: false matrix: profile: - ${{ needs.prepare.outputs.BUILD_PROFILE }} + builder: + - 5.0-26 otp: - 24.3.4.2-1 elixir: @@ -244,6 +246,7 @@ jobs: - name: build emqx packages working-directory: source env: + BUILDER: ${{ matrix.builder }} OTP: ${{ matrix.otp }} ELIXIR: ${{ matrix.elixir }} PROFILE: ${{ matrix.profile }} @@ -272,7 +275,7 @@ jobs: --pkgtype "${PKGTYPE}" \ --arch "${ARCH}" \ --elixir "${IsElixir}" \ - --builder "ghcr.io/emqx/emqx-builder/5.0-24:${ELIXIR}-${OTP}-${SYSTEM}" + --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}" done - uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/build_slim_packages.yaml b/.github/workflows/build_slim_packages.yaml index 12b17535a..c48833453 100644 --- a/.github/workflows/build_slim_packages.yaml +++ b/.github/workflows/build_slim_packages.yaml @@ -29,14 +29,15 @@ jobs: fail-fast: false matrix: profile: - - ["emqx", "el7"] - - ["emqx-enterprise", "ubuntu20.04"] - otp: - - 24.3.4.2-1 + - ["emqx", "24.3.4.2-1", "el7"] + - ["emqx", "25.1.2-2", "ubuntu20.04"] + - ["emqx-enterprise", "24.3.4.2-1", "ubuntu20.04"] + builder: + - 5.0-26 elixir: - 1.13.4 - container: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.profile[1] }}" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" steps: - uses: AutoModality/action-clean@v1 @@ -72,7 +73,7 @@ jobs: ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg - uses: actions/upload-artifact@v3 with: - name: "${{ matrix.profile[0] }}-${{ matrix.otp }}-${{ matrix.profile[1] }}" + name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}" path: _packages/${{ matrix.profile[0] }}/* - uses: actions/upload-artifact@v3 with: @@ -89,7 +90,7 @@ jobs: profile: - emqx otp: - - 24.2.1 + - 25.1.2 steps: - uses: actions/checkout@v3 - uses: ilammy/msvc-dev-cmd@v1.12.0 @@ -119,7 +120,7 @@ jobs: - uses: actions/upload-artifact@v3 with: name: windows - path: _packages/${{ matrix.profile}}/* + path: _packages/${{ matrix.profile }}/* mac: strategy: diff --git a/.github/workflows/check_deps_integrity.yaml b/.github/workflows/check_deps_integrity.yaml index 2578e078f..59fc01d74 100644 --- a/.github/workflows/check_deps_integrity.yaml +++ b/.github/workflows/check_deps_integrity.yaml @@ -5,7 +5,7 @@ on: [pull_request, push] jobs: check_deps_integrity: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04 steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style_check.yaml b/.github/workflows/code_style_check.yaml index 910bb8835..6ab8ea8b6 100644 --- a/.github/workflows/code_style_check.yaml +++ b/.github/workflows/code_style_check.yaml @@ -5,7 +5,7 @@ on: [pull_request] jobs: code_style_check: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04" steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/elixir_apps_check.yaml b/.github/workflows/elixir_apps_check.yaml index e4b93cf80..62000f421 100644 --- a/.github/workflows/elixir_apps_check.yaml +++ b/.github/workflows/elixir_apps_check.yaml @@ -8,7 +8,7 @@ jobs: elixir_apps_check: runs-on: ubuntu-latest # just use the latest builder - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04" strategy: fail-fast: false diff --git a/.github/workflows/elixir_deps_check.yaml b/.github/workflows/elixir_deps_check.yaml index 9f1d6f3e4..5e64d69c4 100644 --- a/.github/workflows/elixir_deps_check.yaml +++ b/.github/workflows/elixir_deps_check.yaml @@ -7,7 +7,7 @@ on: [pull_request, push] jobs: elixir_deps_check: runs-on: ubuntu-20.04 - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04 steps: - name: Checkout diff --git a/.github/workflows/elixir_release.yml b/.github/workflows/elixir_release.yml index 0ac1e9155..f53051d1d 100644 --- a/.github/workflows/elixir_release.yml +++ b/.github/workflows/elixir_release.yml @@ -17,8 +17,7 @@ jobs: profile: - emqx - emqx-enterprise - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04 - + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-25.1.2-2-ubuntu20.04 steps: - name: Checkout uses: actions/checkout@v3 diff --git a/.github/workflows/run_emqx_app_tests.yaml b/.github/workflows/run_emqx_app_tests.yaml index 71b016761..ddbb14609 100644 --- a/.github/workflows/run_emqx_app_tests.yaml +++ b/.github/workflows/run_emqx_app_tests.yaml @@ -11,8 +11,11 @@ jobs: run_emqx_app_tests: strategy: matrix: + builder: + - 5.0-26 otp: - 24.3.4.2-1 + - 25.1.2-2 # no need to use more than 1 version of Elixir, since tests # run using only Erlang code. This is needed just to specify # the base image. @@ -34,7 +37,7 @@ jobs: use-self-hosted: false runs-on: ${{ matrix.runs-on }} - container: "ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}" defaults: run: diff --git a/.github/workflows/run_fvt_tests.yaml b/.github/workflows/run_fvt_tests.yaml index 8870b8257..9e51b7142 100644 --- a/.github/workflows/run_fvt_tests.yaml +++ b/.github/workflows/run_fvt_tests.yaml @@ -16,7 +16,7 @@ jobs: prepare: runs-on: ubuntu-20.04 # prepare source with any OTP version, no need for a matrix - container: ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-alpine3.15.1 + container: ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1 steps: - uses: actions/checkout@v3 @@ -48,6 +48,8 @@ jobs: - rlog os: - ["alpine3.15.1", "alpine:3.15.1"] + builder: + - 5.0-26 otp: - 24.3.4.2-1 elixir: @@ -57,7 +59,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3.4 - uses: actions/download-artifact@v3 with: name: source @@ -68,7 +70,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_RUNNER: ${{ matrix.os[1] }} run: | make ${{ matrix.profile }}-docker @@ -119,6 +121,8 @@ jobs: - emqx-enterprise os: - ["debian11", "debian:11-slim"] + builder: + - 5.0-26 otp: - 24.3.4.2-1 elixir: @@ -130,7 +134,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3.4 - uses: actions/download-artifact@v3 with: name: source @@ -141,7 +145,7 @@ jobs: - name: make docker image working-directory: source env: - EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-24:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} + EMQX_BUILDER: ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }} EMQX_RUNNER: ${{ matrix.os[1] }} run: | make ${{ matrix.profile }}-docker diff --git a/.github/workflows/run_jmeter_tests.yaml b/.github/workflows/run_jmeter_tests.yaml index f67730b30..283d7c014 100644 --- a/.github/workflows/run_jmeter_tests.yaml +++ b/.github/workflows/run_jmeter_tests.yaml @@ -16,7 +16,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - name: download jmeter timeout-minutes: 3 env: @@ -36,7 +36,7 @@ jobs: PROFILE='emqx' make "${PROFILE}-docker" VSN="$(./pkg-vsn.sh $PROFILE)" - echo "::set-output name=version::${VSN}" + echo "version=${VSN}" >> $GITHUB_OUTPUT docker save -o emqx.tar emqx/emqx:${VSN} - uses: actions/upload-artifact@v3 with: @@ -59,7 +59,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -155,7 +155,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -261,7 +261,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -363,7 +363,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: @@ -462,7 +462,7 @@ jobs: steps: - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 - uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/run_relup_tests.yaml b/.github/workflows/run_relup_tests.yaml index ac2da6f85..cb1540fdd 100644 --- a/.github/workflows/run_relup_tests.yaml +++ b/.github/workflows/run_relup_tests.yaml @@ -16,7 +16,7 @@ on: jobs: relup_test_plan: runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" outputs: CUR_EE_VSN: ${{ steps.find-versions.outputs.CUR_EE_VSN }} OLD_VERSIONS: ${{ steps.find-versions.outputs.OLD_VERSIONS }} @@ -37,8 +37,8 @@ jobs: ee_vsn="$(./pkg-vsn.sh enterprise)" old_ee_vsns="$(./scripts/relup-build/base-vsns.sh enterprise | xargs)" old_vsns=$(echo -n "${old_ee_vsns}" | sed 's/ $//g' | jq -R -s -c 'split(" ")') - echo "::set-output name=CUR_EE_VSN::$ee_vsn" - echo "::set-output name=OLD_VERSIONS::$old_vsns" + echo "CUR_EE_VSN=$ee_vsn" >> $GITHUB_OUTPUT + echo "OLD_VERSIONS=$old_vsns" >> $GITHUB_OUTPUT - name: build emqx run: | set -x @@ -74,7 +74,7 @@ jobs: # setup Erlang to run lux - uses: erlef/setup-beam@v1 with: - otp-version: "24.2" + otp-version: 24.3 - uses: actions/checkout@v3 with: repository: hawk/lux diff --git a/.github/workflows/run_test_cases.yaml b/.github/workflows/run_test_cases.yaml index a21f130d4..4ee8efa2e 100644 --- a/.github/workflows/run_test_cases.yaml +++ b/.github/workflows/run_test_cases.yaml @@ -14,49 +14,55 @@ on: pull_request: jobs: + build-matrix: + runs-on: ubuntu-latest + outputs: + prepare: ${{ steps.matrix.outputs.prepare }} + host: ${{ steps.matrix.outputs.host }} + docker: ${{ steps.matrix.outputs.docker }} + runs-on: ${{ steps.runner.outputs.runs-on }} + steps: + - uses: actions/checkout@v3 + - name: Build matrix + id: matrix + run: | + APPS="$(./scripts/find-apps.sh --ci)" + MATRIX="$(echo "${APPS}" | jq -c ' + [ + (.[] | select(.profile == "emqx") | . + { + builder: "5.0-26", + otp: "25.1.2-2", + elixir: "1.13.4" + }), + (.[] | select(.profile == "emqx-enterprise") | . + { + builder: "5.0-26", + otp: ["24.3.4.2-1", "25.1.2-2"][], + elixir: "1.13.4" + }) + ] + ')" + echo "${MATRIX}" | jq + MATRIX_PREPARE="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')" + MATRIX_HOST="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "host"))')" + MATRIX_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')" + echo "prepare=${MATRIX_PREPARE}" | tee -a $GITHUB_OUTPUT + echo "host=${MATRIX_HOST}" | tee -a $GITHUB_OUTPUT + echo "docker=${MATRIX_DOCKER}" | tee -a $GITHUB_OUTPUT + - name: Choose runner host + id: runner + run: | + RUNS_ON="ubuntu-20.04" + ${{ github.repository_owner == 'emqx' }} && RUNS_ON="aws-amd64" + echo "runs-on=${RUNS_ON}" | tee -a $GITHUB_OUTPUT + prepare: runs-on: aws-amd64 - # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" - outputs: - fast_ct_apps: ${{ steps.find_ct_apps.outputs.fast_ct_apps }} - docker_ct_apps: ${{ steps.find_ct_apps.outputs.docker_ct_apps }} - steps: - - uses: AutoModality/action-clean@v1 - - uses: actions/checkout@v3 - with: - path: source - - name: Find CT Apps - working-directory: source - id: find_ct_apps - run: | - fast_ct_apps="$(./scripts/find-apps.sh --ci fast)" - docker_ct_apps="$(./scripts/find-apps.sh --ci docker)" - echo "fast: $fast_ct_apps" - echo "docker: $docker_ct_apps" - echo "::set-output name=fast_ct_apps::$fast_ct_apps" - echo "::set-output name=docker_ct_apps::$docker_ct_apps" - - name: get_all_deps - working-directory: source - env: - PROFILE: emqx - #DIAGNOSTIC: 1 - run: | - make ensure-rebar3 - # fetch all deps and compile - make emqx - make test-compile - cd .. - zip -ryq source.zip source/* source/.[^.]* - - uses: actions/upload-artifact@v3 - with: - name: source-emqx - path: source.zip - - prepare_ee: - runs-on: aws-amd64 - # prepare source with any OTP version, no need for a matrix - container: "ghcr.io/emqx/emqx-builder/5.0-18:1.13.4-24.3.4.2-1-ubuntu20.04" + needs: [build-matrix] + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 @@ -65,61 +71,48 @@ jobs: - name: get_all_deps working-directory: source env: - PROFILE: emqx-enterprise + PROFILE: ${{ matrix.profile }} #DIAGNOSTIC: 1 run: | make ensure-rebar3 # fetch all deps and compile - make emqx-enterprise + make ${{ matrix.profile }} make test-compile cd .. zip -ryq source.zip source/* source/.[^.]* - uses: actions/upload-artifact@v3 with: - name: source-emqx-enterprise + name: source-${{ matrix.profile }}-${{ matrix.otp }} path: source.zip eunit_and_proper: needs: + - build-matrix - prepare - - prepare_ee - runs-on: ${{ matrix.runs-on }} + runs-on: ${{ needs.build-matrix.outputs.runs-on }} strategy: fail-fast: false matrix: - profile: - - emqx - - emqx-enterprise - runs-on: - - aws-amd64 - - ubuntu-20.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - exclude: - - runs-on: ubuntu-20.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false + include: ${{ fromJson(needs.build-matrix.outputs.prepare) }} defaults: run: shell: bash - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.profile }} + name: source-${{ matrix.profile }}-${{ matrix.otp }} path: . - name: unzip source code - env: - PROFILE: ${{ matrix.profile }} run: unzip -o -q source.zip # produces eunit.coverdata - name: eunit env: PROFILE: ${{ matrix.profile }} + CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} working-directory: source run: make eunit @@ -127,6 +120,7 @@ jobs: - name: proper env: PROFILE: ${{ matrix.profile }} + CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} working-directory: source run: make proper @@ -137,23 +131,14 @@ jobs: ct_docker: needs: + - build-matrix - prepare - - prepare_ee + runs-on: ${{ needs.build-matrix.outputs.runs-on }} strategy: fail-fast: false matrix: - app: ${{ fromJson(needs.prepare.outputs.docker_ct_apps) }} - runs-on: - - aws-amd64 - - ubuntu-20.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - exclude: - - runs-on: ubuntu-20.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false - runs-on: ${{ matrix.runs-on }} + include: ${{ fromJson(needs.build-matrix.outputs.docker) }} + defaults: run: shell: bash @@ -162,24 +147,22 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.app[1] }} + name: source-${{ matrix.profile }}-${{ matrix.otp }} path: . - name: unzip source code run: unzip -q source.zip - name: run tests working-directory: source env: + DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" MONGO_TAG: 5 MYSQL_TAG: 8 PGSQL_TAG: 13 REDIS_TAG: 6 INFLUXDB_TAG: 2.5.0 - WHICH_APP: ${{ matrix.app[0] }} - PROFILE: ${{ matrix.app[1] }} - run: | - echo $PROFILE - rm _build/default/lib/rocksdb/_build/cmake/CMakeCache.txt - ./scripts/ct/run.sh --app $WHICH_APP + PROFILE: ${{ matrix.profile }} + CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} + run: ./scripts/ct/run.sh --app ${{ matrix.app }} - uses: actions/upload-artifact@v3 with: name: coverdata @@ -187,30 +170,20 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.app[1] }}-${{ matrix.app[2] }} + name: logs-${{ matrix.profile }}-${{ matrix.prefix }} path: source/_build/test/logs ct: needs: + - build-matrix - prepare - - prepare_ee + runs-on: ${{ needs.build-matrix.outputs.runs-on }} strategy: fail-fast: false matrix: - app: ${{ fromJson(needs.prepare.outputs.fast_ct_apps) }} - runs-on: - - aws-amd64 - - ubuntu-20.04 - use-self-hosted: - - ${{ github.repository_owner == 'emqx' }} - exclude: - - runs-on: ubuntu-20.04 - use-self-hosted: true - - runs-on: aws-amd64 - use-self-hosted: false + include: ${{ fromJson(needs.build-matrix.outputs.host) }} - runs-on: ${{ matrix.runs-on }} - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu20.04" defaults: run: shell: bash @@ -219,7 +192,7 @@ jobs: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-${{ matrix.app[1] }} + name: source-${{ matrix.profile }}-${{ matrix.otp }} path: . - name: unzip source code run: unzip -q source.zip @@ -228,10 +201,10 @@ jobs: - name: run common test working-directory: source env: - WHICH_APP: ${{ matrix.app[0] }} - PROFILE: ${{ matrix.app[1] }} + PROFILE: ${{ matrix.profile }} + CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }} run: | - make "${WHICH_APP}-ct" + make "${{ matrix.app }}-ct" - uses: actions/upload-artifact@v3 with: name: coverdata @@ -240,7 +213,7 @@ jobs: - uses: actions/upload-artifact@v3 if: failure() with: - name: logs-${{ matrix.app[1] }}-${{ matrix.app[2] }} + name: logs-${{ matrix.profile }}-${{ matrix.prefix }} path: source/_build/test/logs make_cover: @@ -249,12 +222,12 @@ jobs: - ct - ct_docker runs-on: ubuntu-20.04 - container: "ghcr.io/emqx/emqx-builder/5.0-24:1.13.4-24.3.4.2-1-ubuntu20.04" + container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" steps: - uses: AutoModality/action-clean@v1 - uses: actions/download-artifact@v3 with: - name: source-emqx-enterprise + name: source-emqx-enterprise-24.3.4.2-1 path: . - name: unzip source code run: unzip -q source.zip diff --git a/Makefile b/Makefile index c5c73995f..201ff9da7 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ REBAR = $(CURDIR)/rebar3 BUILD = $(CURDIR)/build SCRIPTS = $(CURDIR)/scripts export EMQX_RELUP ?= true -export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian11 +export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 export EMQX_DEFAULT_RUNNER = debian:11-slim export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh) export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh) @@ -24,6 +24,7 @@ PROFILES := $(REL_PROFILES) $(PKG_PROFILES) default CT_NODE_NAME ?= 'test@127.0.0.1' CT_READABLE ?= true +CT_COVER_EXPORT_PREFIX ?= $(PROFILE) export REBAR_GIT_CLONE_OPTIONS += --depth=1 @@ -62,7 +63,7 @@ mix-deps-get: $(ELIXIR_COMMON_DEPS) .PHONY: eunit eunit: $(REBAR) merge-config - @ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c --cover_export_name $(PROFILE)-eunit + @ENABLE_COVER_COMPILE=1 $(REBAR) eunit -v -c --cover_export_name $(CT_COVER_EXPORT_PREFIX)-eunit .PHONY: proper proper: $(REBAR) @@ -74,7 +75,7 @@ test-compile: $(REBAR) merge-config .PHONY: ct ct: $(REBAR) merge-config - @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(PROFILE)-ct + @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(CT_COVER_EXPORT_PREFIX)-ct .PHONY: static_checks static_checks: @@ -86,7 +87,10 @@ APPS=$(shell $(SCRIPTS)/find-apps.sh) define gen-app-ct-target $1-ct: $(REBAR) @$(SCRIPTS)/pre-compile.sh $(PROFILE) - @ENABLE_COVER_COMPILE=1 $(REBAR) ct --name $(CT_NODE_NAME) -c -v --cover_export_name $(PROFILE)-$(subst /,-,$1) --suite $(shell $(SCRIPTS)/find-suites.sh $1) + @ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \ + --name $(CT_NODE_NAME) \ + --cover_export_name $(CT_COVER_EXPORT_PREFIX)-$(subst /,-,$1) \ + --suite $(shell $(SCRIPTS)/find-suites.sh $1) endef $(foreach app,$(APPS),$(eval $(call gen-app-ct-target,$(app)))) @@ -238,8 +242,8 @@ define gen-elixir-pkg-target # the Elixir places the tar in a different path than Rebar3 $1-elixir-pkg: $(COMMON_DEPS) @env TAR_PKG_DIR=_build/$1-pkg \ - IS_ELIXIR=yes \ - $(BUILD) $1-pkg pkg + IS_ELIXIR=yes \ + $(BUILD) $1-pkg pkg endef $(foreach pt,$(REL_PROFILES),$(eval $(call gen-elixir-pkg-target,$(pt)))) diff --git a/apps/emqx/rebar.config b/apps/emqx/rebar.config index ff27566ec..4c00d7f23 100644 --- a/apps/emqx/rebar.config +++ b/apps/emqx/rebar.config @@ -50,7 +50,7 @@ ]}. {dialyzer, [ - {warnings, [unmatched_returns, error_handling, race_conditions]}, + {warnings, [unmatched_returns, error_handling]}, {plt_location, "."}, {plt_prefix, "emqx_dialyzer"}, {plt_apps, all_apps}, diff --git a/apps/emqx/src/emqx_connection.erl b/apps/emqx/src/emqx_connection.erl index d892f5acc..d604d4784 100644 --- a/apps/emqx/src/emqx_connection.erl +++ b/apps/emqx/src/emqx_connection.erl @@ -641,6 +641,7 @@ close_socket_ok(State) -> ok. %% tell truth about the original exception +-spec maybe_raise_exception(any()) -> no_return(). maybe_raise_exception(#{ exception := Exception, context := Context, diff --git a/apps/emqx/src/emqx_tls_lib.erl b/apps/emqx/src/emqx_tls_lib.erl index 85c4396ab..fc3ed9353 100644 --- a/apps/emqx/src/emqx_tls_lib.erl +++ b/apps/emqx/src/emqx_tls_lib.erl @@ -55,18 +55,6 @@ %% non-empty list of strings -define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))). -%% The ciphers that ssl:cipher_suites(exclusive, 'tlsv1.3', openssl) -%% should return when running on otp 23. -%% But we still have to hard-code them because tlsv1.3 on otp 22 is -%% not trustworthy. --define(TLSV13_EXCLUSIVE_CIPHERS, [ - "TLS_AES_256_GCM_SHA384", - "TLS_AES_128_GCM_SHA256", - "TLS_CHACHA20_POLY1305_SHA256", - "TLS_AES_128_CCM_SHA256", - "TLS_AES_128_CCM_8_SHA256" -]). - -define(SELECTED_CIPHERS, [ "ECDHE-ECDSA-AES256-GCM-SHA384", "ECDHE-RSA-AES256-GCM-SHA384", @@ -162,7 +150,7 @@ all_ciphers(['tlsv1.3']) -> %% because 'all' returns legacy cipher suites too, %% which does not make sense since tlsv1.3 can not use %% legacy cipher suites. - ?TLSV13_EXCLUSIVE_CIPHERS; + ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); all_ciphers(Versions) -> %% assert non-empty List = lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions]), @@ -197,7 +185,7 @@ selected_ciphers(Vsns) -> do_selected_ciphers('tlsv1.3') -> case lists:member('tlsv1.3', proplists:get_value(available, ssl:versions())) of - true -> ?TLSV13_EXCLUSIVE_CIPHERS; + true -> ssl:cipher_suites(exclusive, 'tlsv1.3', openssl); false -> [] end ++ do_selected_ciphers('tlsv1.2'); do_selected_ciphers(_) -> diff --git a/apps/emqx/test/emqx_common_test_helpers.erl b/apps/emqx/test/emqx_common_test_helpers.erl index 1d0d793bd..ac8659735 100644 --- a/apps/emqx/test/emqx_common_test_helpers.erl +++ b/apps/emqx/test/emqx_common_test_helpers.erl @@ -16,8 +16,6 @@ -module(emqx_common_test_helpers). --include_lib("common_test/include/ct.hrl"). - -type special_config_handler() :: fun(). -type apps() :: list(atom()). @@ -78,12 +76,6 @@ -define(CERTS_PATH(CertName), filename:join(["etc", "certs", CertName])). --define(MQTT_SSL_TWOWAY, [ - {cacertfile, ?CERTS_PATH("cacert.pem")}, - {verify, verify_peer}, - {fail_if_no_peer_cert, true} -]). - -define(MQTT_SSL_CLIENT_CERTS, [ {keyfile, ?CERTS_PATH("client-key.pem")}, {cacertfile, ?CERTS_PATH("cacert.pem")}, diff --git a/apps/emqx/test/emqx_router_helper_SUITE.erl b/apps/emqx/test/emqx_router_helper_SUITE.erl index 875ec8b6c..20c94087a 100644 --- a/apps/emqx/test/emqx_router_helper_SUITE.erl +++ b/apps/emqx/test/emqx_router_helper_SUITE.erl @@ -19,7 +19,6 @@ -compile(export_all). -compile(nowarn_export_all). --include("emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). -include_lib("common_test/include/ct.hrl"). -include_lib("snabbkaffe/include/snabbkaffe.hrl"). diff --git a/apps/emqx/test/emqx_shared_sub_SUITE.erl b/apps/emqx/test/emqx_shared_sub_SUITE.erl index 7c38fdf17..a428a6fa9 100644 --- a/apps/emqx/test/emqx_shared_sub_SUITE.erl +++ b/apps/emqx/test/emqx_shared_sub_SUITE.erl @@ -21,8 +21,6 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). -define(SUITE, ?MODULE). diff --git a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl index a3787fe6e..fcf65046d 100644 --- a/apps/emqx_conf/test/emqx_conf_app_SUITE.erl +++ b/apps/emqx_conf/test/emqx_conf_app_SUITE.erl @@ -19,10 +19,7 @@ -compile(export_all). -compile(nowarn_export_all). --include("emqx_conf.hrl"). -include_lib("eunit/include/eunit.hrl"). --include_lib("common_test/include/ct.hrl"). --include_lib("snabbkaffe/include/snabbkaffe.hrl"). all() -> emqx_common_test_helpers:all(?MODULE). diff --git a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl index c925aca20..cb975ca63 100644 --- a/apps/emqx_connector/src/emqx_connector_jwt_worker.erl +++ b/apps/emqx_connector/src/emqx_connector_jwt_worker.erl @@ -154,8 +154,8 @@ handle_info({timeout, TRef, ?refresh_jwt}, State0 = #{refresh_timer := TRef}) -> handle_info(_Msg, State) -> {noreply, State}. -format_status(State) -> - censor_secrets(State). +format_status(Status = #{state := State}) -> + Status#{state => censor_secrets(State)}. format_status(_Opt, [_PDict, State0]) -> State = censor_secrets(State0), @@ -222,16 +222,13 @@ ensure_timer(State) -> State. -spec censor_secrets(state()) -> map(). -censor_secrets(State) -> - maps:map( - fun - (Key, _Value) when - Key =:= jwt; - Key =:= jwk - -> - "******"; - (_Key, Value) -> - Value - end, - State - ). +censor_secrets(State = #{jwt := JWT, jwk := JWK}) -> + State#{ + jwt := censor_secret(JWT), + jwk := censor_secret(JWK) + }. + +censor_secret(undefined) -> + undefined; +censor_secret(_Secret) -> + "******". diff --git a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl index d73cb79fd..d0f22f134 100644 --- a/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl +++ b/apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl @@ -209,20 +209,27 @@ t_log_file(_Config) -> ?assertMatch([#{<<"mtime">> := _, <<"size">> := _, <<"node">> := _}], json(Detail)), {ok, Binary} = request_api(get, api_path("trace/test_client_id/download"), Header), {ok, [ - _Comment, + Comment, #zip_file{ name = ZipName, info = #file_info{size = Size, type = regular, access = read_write} } - ]} = - ZipTab = - zip:table(Binary), + ]} = zip:table(Binary), ?assert(Size > 0), ZipNamePrefix = lists:flatten(io_lib:format("~s-trace_~s", [node(), Name])), ?assertNotEqual(nomatch, re:run(ZipName, [ZipNamePrefix])), Path = api_path("trace/test_client_id/download?node=" ++ atom_to_list(node())), {ok, Binary2} = request_api(get, Path, Header), - ?assertEqual(ZipTab, zip:table(Binary2)), + ?assertMatch( + {ok, [ + Comment, + #zip_file{ + name = ZipName, + info = #file_info{size = Size, type = regular, access = read_write} + } + ]}, + zip:table(Binary2) + ), {error, {_, 400, _}, _} = request_api( get, diff --git a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl index 39eea5fc2..e933a36d4 100644 --- a/apps/emqx_plugins/test/emqx_plugins_SUITE.erl +++ b/apps/emqx_plugins/test/emqx_plugins_SUITE.erl @@ -22,7 +22,7 @@ -include_lib("emqx/include/emqx.hrl"). -include_lib("eunit/include/eunit.hrl"). --define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0-rc.3"). +-define(EMQX_PLUGIN_TEMPLATE_VSN, "5.0.0"). -define(EMQX_ELIXIR_PLUGIN_TEMPLATE_VSN, "0.1.0"). -define(PACKAGE_SUFFIX, ".tar.gz"). diff --git a/apps/emqx_resource/rebar.config b/apps/emqx_resource/rebar.config index e6857829f..929514ddd 100644 --- a/apps/emqx_resource/rebar.config +++ b/apps/emqx_resource/rebar.config @@ -10,10 +10,6 @@ {extra_src_dirs, ["examples"]}. -%% try to override the dialyzer 'race_conditions' defined in the top-level dir, -%% but it doesn't work -{dialyzer, [{warnings, [unmatched_returns, error_handling]}]}. - {deps, [ {jsx, {git, "https://github.com/talentdeficit/jsx", {tag, "v3.1.0"}}}, {emqx, {path, "../emqx"}} diff --git a/deploy/docker/Dockerfile b/deploy/docker/Dockerfile index 51b4dbd0c..4a00c68fb 100644 --- a/deploy/docker/Dockerfile +++ b/deploy/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian11 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 ARG RUN_FROM=debian:11-slim FROM ${BUILD_FROM} AS builder diff --git a/deploy/docker/Dockerfile.alpine b/deploy/docker/Dockerfile.alpine index dc16afc54..0f72be9ab 100644 --- a/deploy/docker/Dockerfile.alpine +++ b/deploy/docker/Dockerfile.alpine @@ -1,4 +1,4 @@ -ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-15:1.13.3-24.2.1-1-alpine3.15.1 +ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-alpine3.15.1 ARG RUN_FROM=alpine:3.15.1 FROM ${BUILD_FROM} AS builder diff --git a/mix.exs b/mix.exs index 11ef14eb1..86bc540ca 100644 --- a/mix.exs +++ b/mix.exs @@ -45,13 +45,14 @@ defmodule EMQXUmbrella.MixProject do # other exact versions, and not ranges. [ {:lc, github: "emqx/lc", tag: "0.3.2", override: true}, - {:redbug, "2.0.7"}, + {:redbug, "2.0.8"}, {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true}, {:ehttpc, github: "emqx/ehttpc", tag: "0.4.2", override: true}, {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true}, {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true}, {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true}, {:esockd, github: "emqx/esockd", tag: "5.9.4", override: true}, + {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-7", override: true}, {:ekka, github: "emqx/ekka", tag: "0.13.7", override: true}, {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true}, {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true}, diff --git a/rebar.config b/rebar.config index e9678b1cd..40614a443 100644 --- a/rebar.config +++ b/rebar.config @@ -45,7 +45,7 @@ {deps, [ {lc, {git, "https://github.com/emqx/lc.git", {tag, "0.3.2"}}} - , {redbug, "2.0.7"} + , {redbug, "2.0.8"} , {gpb, "4.19.5"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps , {typerefl, {git, "https://github.com/ieQu1/typerefl", {tag, "0.9.1"}}} , {gun, {git, "https://github.com/emqx/gun", {tag, "1.3.9"}}} @@ -54,6 +54,7 @@ , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}} , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}} , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.4"}}} + , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-7"}}} , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.7"}}} , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}} , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}} diff --git a/rebar.config.erl b/rebar.config.erl index 8c8f87fa8..f40c86a48 100644 --- a/rebar.config.erl +++ b/rebar.config.erl @@ -15,8 +15,8 @@ do(Dir, CONFIG) -> end. assert_otp() -> - Oldest = 23, - Latest = 24, + Oldest = 24, + Latest = 25, OtpRelease = list_to_integer(erlang:system_info(otp_release)), case OtpRelease < Oldest orelse OtpRelease > Latest of true -> @@ -119,7 +119,7 @@ is_win32() -> win32 =:= element(1, os:type()). project_app_dirs() -> - project_app_dirs(get_edition_from_profille_env()). + project_app_dirs(get_edition_from_profile_env()). project_app_dirs(Edition) -> ["apps/*"] ++ @@ -154,14 +154,14 @@ test_deps() -> {erl_csv, "0.2.0"} ]. -common_compile_opts(Vsn) -> - common_compile_opts(get_edition_from_profille_env(), Vsn). +common_compile_opts() -> + common_compile_opts(get_edition_from_profile_env(), undefined). common_compile_opts(Edition, Vsn) -> % always include debug_info [ debug_info, - {compile_info, [{emqx_vsn, Vsn}]}, + {compile_info, [{emqx_vsn, Vsn} || Vsn /= undefined]}, {d, 'EMQX_RELEASE_EDITION', Edition} ] ++ [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"] ++ @@ -180,7 +180,7 @@ warn_profile_env() -> end. %% this function is only used for test/check profiles -get_edition_from_profille_env() -> +get_edition_from_profile_env() -> case os:getenv("PROFILE") of "emqx-enterprise" ++ _ -> ee; @@ -243,15 +243,14 @@ profiles_ee() -> %% EE has more files than CE, always test/check with EE options. profiles_dev() -> - Vsn = get_vsn('emqx-enterprise'), [ {check, [ - {erl_opts, common_compile_opts(Vsn)}, + {erl_opts, common_compile_opts()}, {project_app_dirs, project_app_dirs()} ]}, {test, [ {deps, test_deps()}, - {erl_opts, common_compile_opts(Vsn) ++ erl_opts_i()}, + {erl_opts, common_compile_opts() ++ erl_opts_i()}, {extra_src_dirs, [{"test", [{recursive, true}]}]}, {project_app_dirs, project_app_dirs()} ]} @@ -270,7 +269,7 @@ relx(Vsn, RelType, PkgType, Edition) -> {vm_args, false}, {release, {emqx, Vsn}, relx_apps(RelType, Edition)}, {overlay, relx_overlay(RelType, Edition)}, - {overlay_vars, + {overlay_vars_values, build_info() ++ [ {emqx_description, emqx_description(RelType, Edition)} diff --git a/scripts/buildx.sh b/scripts/buildx.sh index 96e48d9eb..ca7f812f6 100755 --- a/scripts/buildx.sh +++ b/scripts/buildx.sh @@ -9,7 +9,7 @@ ## example: ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \ -## --builder ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian10 +## --builder ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11 set -euo pipefail @@ -22,9 +22,9 @@ help() { echo "--elixir: Specify if the release should be built with Elixir, " echo " defaults to 'no'." echo "--arch amd64|arm64: Target arch to build the EMQX package for" - echo "--src_dir : EMQX source ode in this dir, default to PWD" + echo "--src_dir : EMQX source code in this dir, default to PWD" echo "--builder : Builder image to pull" - echo " E.g. ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-debian10" + echo " E.g. ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-debian11" } while [ "$#" -gt 0 ]; do diff --git a/scripts/ct/run.sh b/scripts/ct/run.sh index ae779c572..d86ee4f6a 100755 --- a/scripts/ct/run.sh +++ b/scripts/ct/run.sh @@ -66,7 +66,7 @@ if [ "${WHICH_APP}" = 'novalue' ]; then exit 1 fi -ERLANG_CONTAINER='erlang24' +ERLANG_CONTAINER='erlang' DOCKER_CT_ENVS_FILE="${WHICH_APP}/docker-ct" case "${WHICH_APP}" in @@ -89,7 +89,7 @@ FILES=( ) for dep in ${CT_DEPS}; do case "${dep}" in - erlang24) + erlang) FILES+=( '.ci/docker-compose-file/docker-compose.yaml' ) ;; toxiproxy) diff --git a/scripts/elvis-check.sh b/scripts/elvis-check.sh index bf5cf9b8e..fcc0be03b 100755 --- a/scripts/elvis-check.sh +++ b/scripts/elvis-check.sh @@ -5,7 +5,7 @@ set -euo pipefail -elvis_version='1.0.0-emqx-2' +elvis_version='1.1.0-emqx-2' base="${1:-}" repo="${2:-emqx/emqx}" diff --git a/scripts/ensure-rebar3.sh b/scripts/ensure-rebar3.sh index e31dba207..1d6794bda 100755 --- a/scripts/ensure-rebar3.sh +++ b/scripts/ensure-rebar3.sh @@ -2,7 +2,7 @@ set -euo pipefail -## rebar3 tag 3.18.0-emqx-1 is compiled using otp24.1.5. +## rebar3 tag 3.19.0-emqx-1 is compiled using latest official OTP-24 image. ## we have to use an otp24-compiled rebar3 because the defination of record #application{} ## in systools.hrl is changed in otp24. OTP_VSN="${OTP_VSN:-$(./scripts/get-otp-vsn.sh)}" @@ -13,6 +13,9 @@ case ${OTP_VSN} in 24*) VERSION="3.18.0-emqx-1" ;; + 25*) + VERSION="3.19.0-emqx-1" + ;; *) echo "Unsupporetd Erlang/OTP version $OTP_VSN" exit 1 diff --git a/scripts/find-apps.sh b/scripts/find-apps.sh index a92c56748..f07cd2f7d 100755 --- a/scripts/find-apps.sh +++ b/scripts/find-apps.sh @@ -8,10 +8,10 @@ cd -P -- "$(dirname -- "$0")/.." help() { echo echo "-h|--help: To display this usage info" - echo "--ci fast|docker: Print apps in json format for github ci mtrix" + echo "--ci: Print apps in json format for github ci matrix" } -CI='novalue' +MODE='list' while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -19,8 +19,8 @@ while [ "$#" -gt 0 ]; do exit 0 ;; --ci) - CI="$2" - shift 2 + MODE='ci' + shift 1 ;; *) echo "unknown option $1" @@ -45,7 +45,7 @@ CE="$(find_app 'apps')" EE="$(find_app 'lib-ee')" APPS_ALL="$(echo -e "${CE}\n${EE}")" -if [ "$CI" = 'novalue' ]; then +if [ "$MODE" = 'list' ]; then echo "${APPS_ALL}" exit 0 fi @@ -54,16 +54,22 @@ fi ###### now deal with the github action's matrix. ################################################## -dimensions() { +format_app_description() { + ## prefix is for github actions (they don't like slash in variables) + local prefix=${1//\//_} + echo -n -e "$( +cat <, , apps_ - ## third one is for github actions (they don't like slash in variables) - echo -n -e "[\"$app\", \"$profile\", \"${app//\//_}\"]" + format_app_description "$app" "$profile" "$runner" } matrix() { - first_row='yes' + local sep='[' for app in ${APPS_ALL}; do - row="$(dimensions "$app")" + row="$(describe_app "$app")" if [ -z "$row" ]; then continue fi - if [ "$first_row" = 'yes' ]; then - first_row='no' - echo -n "$row" - else - echo -n ",${row}" - fi + echo -n "${sep}${row}" + sep=', ' done + echo ']' } -echo -n '[' matrix -echo ']' diff --git a/scripts/relup-test/run-relup-lux.sh b/scripts/relup-test/run-relup-lux.sh index d446ee29f..cf30db850 100755 --- a/scripts/relup-test/run-relup-lux.sh +++ b/scripts/relup-test/run-relup-lux.sh @@ -45,8 +45,8 @@ fi # From now on, no need for the v|e prefix OLD_VSN="${old_vsn#[e|v]}" -OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz" -CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.2.1-1-ubuntu20.04-amd64.tar.gz" +OLD_PKG="$(pwd)/_upgrade_base/${profile}-${OLD_VSN}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz" +CUR_PKG="$(pwd)/_packages/${profile}/${profile}-${cur_vsn}-otp24.3.4.2-1-ubuntu20.04-amd64.tar.gz" if [ ! -f "$OLD_PKG" ]; then echo "$OLD_PKG not found" diff --git a/scripts/relup-test/start-relup-test-cluster.sh b/scripts/relup-test/start-relup-test-cluster.sh index b66894c95..c9e3edbd9 100755 --- a/scripts/relup-test/start-relup-test-cluster.sh +++ b/scripts/relup-test/start-relup-test-cluster.sh @@ -22,7 +22,7 @@ WEBHOOK="webhook.$NET" BENCH="bench.$NET" COOKIE='this-is-a-secret' ## Erlang image is needed to run webhook server and emqtt-bench -ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-ubuntu20.04" +ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04" # builder has emqtt-bench installed BENCH_IMAGE="$ERLANG_IMAGE"